View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.lang.Thread.UncaughtExceptionHandler;
24  import java.lang.annotation.Retention;
25  import java.lang.annotation.RetentionPolicy;
26  import java.lang.management.ManagementFactory;
27  import java.lang.management.MemoryUsage;
28  import java.lang.reflect.Constructor;
29  import java.net.BindException;
30  import java.net.InetSocketAddress;
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.HashMap;
36  import java.util.HashSet;
37  import java.util.Iterator;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.Map.Entry;
41  import java.util.Random;
42  import java.util.Set;
43  import java.util.SortedMap;
44  import java.util.TreeMap;
45  import java.util.TreeSet;
46  import java.util.concurrent.ConcurrentHashMap;
47  import java.util.concurrent.ConcurrentMap;
48  import java.util.concurrent.ConcurrentSkipListMap;
49  import java.util.concurrent.locks.ReentrantReadWriteLock;
50  
51  import javax.management.ObjectName;
52  
53  import com.google.protobuf.HBaseZeroCopyByteString;
54  import org.apache.commons.logging.Log;
55  import org.apache.commons.logging.LogFactory;
56  import org.apache.hadoop.classification.InterfaceAudience;
57  import org.apache.hadoop.conf.Configuration;
58  import org.apache.hadoop.fs.FileSystem;
59  import org.apache.hadoop.fs.Path;
60  import org.apache.hadoop.hbase.Cell;
61  import org.apache.hadoop.hbase.CellScannable;
62  import org.apache.hadoop.hbase.CellScanner;
63  import org.apache.hadoop.hbase.CellUtil;
64  import org.apache.hadoop.hbase.Chore;
65  import org.apache.hadoop.hbase.ClockOutOfSyncException;
66  import org.apache.hadoop.hbase.DoNotRetryIOException;
67  import org.apache.hadoop.hbase.HBaseConfiguration;
68  import org.apache.hadoop.hbase.HBaseIOException;
69  import org.apache.hadoop.hbase.HConstants;
70  import org.apache.hadoop.hbase.HRegionInfo;
71  import org.apache.hadoop.hbase.HTableDescriptor;
72  import org.apache.hadoop.hbase.HealthCheckChore;
73  import org.apache.hadoop.hbase.KeyValueUtil;
74  import org.apache.hadoop.hbase.NotServingRegionException;
75  import org.apache.hadoop.hbase.RemoteExceptionHandler;
76  import org.apache.hadoop.hbase.ServerName;
77  import org.apache.hadoop.hbase.Stoppable;
78  import org.apache.hadoop.hbase.TableDescriptors;
79  import org.apache.hadoop.hbase.TableName;
80  import org.apache.hadoop.hbase.UnknownScannerException;
81  import org.apache.hadoop.hbase.YouAreDeadException;
82  import org.apache.hadoop.hbase.ZNodeClearer;
83  import org.apache.hadoop.hbase.catalog.CatalogTracker;
84  import org.apache.hadoop.hbase.catalog.MetaEditor;
85  import org.apache.hadoop.hbase.catalog.MetaReader;
86  import org.apache.hadoop.hbase.client.Append;
87  import org.apache.hadoop.hbase.client.Delete;
88  import org.apache.hadoop.hbase.client.Get;
89  import org.apache.hadoop.hbase.client.HConnectionManager;
90  import org.apache.hadoop.hbase.client.Increment;
91  import org.apache.hadoop.hbase.client.Mutation;
92  import org.apache.hadoop.hbase.client.Put;
93  import org.apache.hadoop.hbase.client.Result;
94  import org.apache.hadoop.hbase.client.RowMutations;
95  import org.apache.hadoop.hbase.client.Scan;
96  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
97  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
98  import org.apache.hadoop.hbase.exceptions.OperationConflictException;
99  import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
100 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
101 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
102 import org.apache.hadoop.hbase.executor.ExecutorService;
103 import org.apache.hadoop.hbase.executor.ExecutorType;
104 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
105 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
106 import org.apache.hadoop.hbase.fs.HFileSystem;
107 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
108 import org.apache.hadoop.hbase.io.hfile.HFile;
109 import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
110 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
111 import org.apache.hadoop.hbase.ipc.PriorityFunction;
112 import org.apache.hadoop.hbase.ipc.RpcCallContext;
113 import org.apache.hadoop.hbase.ipc.RpcClient;
114 import org.apache.hadoop.hbase.ipc.RpcServer;
115 import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
116 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
117 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
118 import org.apache.hadoop.hbase.ipc.ServerRpcController;
119 import org.apache.hadoop.hbase.master.SplitLogManager;
120 import org.apache.hadoop.hbase.master.TableLockManager;
121 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
122 import org.apache.hadoop.hbase.protobuf.RequestConverter;
123 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
124 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
125 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
127 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionResponse;
129 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionResponse;
131 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
132 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
133 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
135 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
136 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
137 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
138 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
139 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
140 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
141 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
142 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
143 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
144 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
145 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
146 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
147 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
148 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
149 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
150 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionResponse;
151 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
152 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerResponse;
153 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
154 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
155 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WALEntry;
156 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
157 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
158 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath;
159 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
160 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Condition;
161 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
162 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
163 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
164 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
165 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
166 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
167 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
168 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
169 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
170 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
171 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
172 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
173 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
174 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
175 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
176 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
177 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
178 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
179 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
180 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
181 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
182 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
183 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
184 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
185 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
186 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
187 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
188 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
189 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
190 import org.apache.hadoop.hbase.regionserver.HRegion.Operation;
191 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
192 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
193 import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
194 import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
195 import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
196 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
197 import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
198 import org.apache.hadoop.hbase.regionserver.wal.HLog;
199 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
200 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
201 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
202 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
203 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
204 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
205 import org.apache.hadoop.hbase.security.UserProvider;
206 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
207 import org.apache.hadoop.hbase.util.Bytes;
208 import org.apache.hadoop.hbase.util.CompressionTest;
209 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
210 import org.apache.hadoop.hbase.util.FSTableDescriptors;
211 import org.apache.hadoop.hbase.util.FSUtils;
212 import org.apache.hadoop.hbase.util.InfoServer;
213 import org.apache.hadoop.hbase.util.JvmPauseMonitor;
214 import org.apache.hadoop.hbase.util.Pair;
215 import org.apache.hadoop.hbase.util.Sleeper;
216 import org.apache.hadoop.hbase.util.Strings;
217 import org.apache.hadoop.hbase.util.Threads;
218 import org.apache.hadoop.hbase.util.VersionInfo;
219 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
220 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
221 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
222 import org.apache.hadoop.hbase.zookeeper.RecoveringRegionWatcher;
223 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
224 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
225 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
226 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
227 import org.apache.hadoop.ipc.RemoteException;
228 import org.apache.hadoop.metrics.util.MBeanUtil;
229 import org.apache.hadoop.net.DNS;
230 import org.apache.hadoop.util.ReflectionUtils;
231 import org.apache.hadoop.util.StringUtils;
232 import org.apache.zookeeper.KeeperException;
233 import org.apache.zookeeper.data.Stat;
234 import org.cliffc.high_scale_lib.Counter;
235 
236 import com.google.protobuf.BlockingRpcChannel;
237 import com.google.protobuf.ByteString;
238 import com.google.protobuf.Message;
239 import com.google.protobuf.RpcController;
240 import com.google.protobuf.ServiceException;
241 import com.google.protobuf.TextFormat;
242 
243 /**
244  * HRegionServer makes a set of HRegions available to clients. It checks in with
245  * the HMaster. There are many HRegionServers in a single HBase deployment.
246  */
247 @InterfaceAudience.Private
248 @SuppressWarnings("deprecation")
249 public class HRegionServer implements ClientProtos.ClientService.BlockingInterface,
250   AdminProtos.AdminService.BlockingInterface, Runnable, RegionServerServices,
251   HBaseRPCErrorHandler, LastSequenceId {
252 
253   public static final Log LOG = LogFactory.getLog(HRegionServer.class);
254 
255   private final Random rand;
256 
257   /*
258    * Strings to be used in forming the exception message for
259    * RegionsAlreadyInTransitionException.
260    */
261   protected static final String OPEN = "OPEN";
262   protected static final String CLOSE = "CLOSE";
263 
264   //RegionName vs current action in progress
265   //true - if open region action in progress
266   //false - if close region action in progress
267   protected final ConcurrentMap<byte[], Boolean> regionsInTransitionInRS =
268     new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
269 
270   /** RPC scheduler to use for the region server. */
271   public static final String REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS =
272       "hbase.region.server.rpc.scheduler.factory.class";
273 
274   protected long maxScannerResultSize;
275 
276   // Cache flushing
277   protected MemStoreFlusher cacheFlusher;
278 
279   // catalog tracker
280   protected CatalogTracker catalogTracker;
281 
282   // Watch if a region is out of recovering state from ZooKeeper
283   @SuppressWarnings("unused")
284   private RecoveringRegionWatcher recoveringRegionWatcher;
285 
286   /**
287    * Go here to get table descriptors.
288    */
289   protected TableDescriptors tableDescriptors;
290 
291   // Replication services. If no replication, this handler will be null.
292   protected ReplicationSourceService replicationSourceHandler;
293   protected ReplicationSinkService replicationSinkHandler;
294 
295   // Compactions
296   public CompactSplitThread compactSplitThread;
297 
298   final ConcurrentHashMap<String, RegionScannerHolder> scanners =
299       new ConcurrentHashMap<String, RegionScannerHolder>();
300 
301   /**
302    * Map of regions currently being served by this region server. Key is the
303    * encoded region name.  All access should be synchronized.
304    */
305   protected final Map<String, HRegion> onlineRegions =
306     new ConcurrentHashMap<String, HRegion>();
307 
308   /**
309    * Map of encoded region names to the DataNode locations they should be hosted on
310    * We store the value as InetSocketAddress since this is used only in HDFS
311    * API (create() that takes favored nodes as hints for placing file blocks).
312    * We could have used ServerName here as the value class, but we'd need to
313    * convert it to InetSocketAddress at some point before the HDFS API call, and
314    * it seems a bit weird to store ServerName since ServerName refers to RegionServers
315    * and here we really mean DataNode locations.
316    */
317   protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap =
318       new ConcurrentHashMap<String, InetSocketAddress[]>();
319 
320   /**
321    * Set of regions currently being in recovering state which means it can accept writes(edits from
322    * previous failed region server) but not reads. A recovering region is also an online region.
323    */
324   protected final Map<String, HRegion> recoveringRegions = Collections
325       .synchronizedMap(new HashMap<String, HRegion>());
326 
327   // Leases
328   protected Leases leases;
329 
330   // Instance of the hbase executor service.
331   protected ExecutorService service;
332 
333   // Request counter. (Includes requests that are not serviced by regions.)
334   final Counter requestCount = new Counter();
335 
336   // If false, the file system has become unavailable
337   protected volatile boolean fsOk;
338   protected HFileSystem fs;
339 
340   // Set when a report to the master comes back with a message asking us to
341   // shutdown. Also set by call to stop when debugging or running unit tests
342   // of HRegionServer in isolation.
343   protected volatile boolean stopped = false;
344 
345   // Go down hard. Used if file system becomes unavailable and also in
346   // debugging and unit tests.
347   protected volatile boolean abortRequested;
348 
349   // region server static info like info port
350   private RegionServerInfo.Builder rsInfo;
351 
352   ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
353 
354   // A state before we go into stopped state.  At this stage we're closing user
355   // space regions.
356   private boolean stopping = false;
357 
358   private volatile boolean killed = false;
359 
360   protected final Configuration conf;
361 
362   private Path rootDir;
363 
364   protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
365 
366   final int numRetries;
367   protected final int threadWakeFrequency;
368   private final int msgInterval;
369 
370   protected final int numRegionsToReport;
371 
372   // Stub to do region server status calls against the master.
373   private RegionServerStatusService.BlockingInterface rssStub;
374   // RPC client. Used to make the stub above that does region server status checking.
375   RpcClient rpcClient;
376 
377   // Server to handle client requests. Default access so can be accessed by
378   // unit tests.
379   RpcServerInterface rpcServer;
380 
381   private final InetSocketAddress isa;
382   private UncaughtExceptionHandler uncaughtExceptionHandler;
383 
384   // Info server. Default access so can be used by unit tests. REGIONSERVER
385   // is name of the webapp and the attribute name used stuffing this instance
386   // into web context.
387   InfoServer infoServer;
388   private JvmPauseMonitor pauseMonitor;
389 
390   /** region server process name */
391   public static final String REGIONSERVER = "regionserver";
392 
393   /** region server configuration name */
394   public static final String REGIONSERVER_CONF = "regionserver_conf";
395 
396   private MetricsRegionServer metricsRegionServer;
397   private SpanReceiverHost spanReceiverHost;
398 
399   /*
400    * Check for compactions requests.
401    */
402   Chore compactionChecker;
403 
404   /*
405    * Check for flushes
406    */
407   Chore periodicFlusher;
408 
409   // HLog and HLog roller. log is protected rather than private to avoid
410   // eclipse warning when accessed by inner classes
411   protected volatile HLog hlog;
412   // The meta updates are written to a different hlog. If this
413   // regionserver holds meta regions, then this field will be non-null.
414   protected volatile HLog hlogForMeta;
415 
416   LogRoller hlogRoller;
417   LogRoller metaHLogRoller;
418 
419   // flag set after we're done setting up server threads (used for testing)
420   protected volatile boolean isOnline;
421 
422   // zookeeper connection and watcher
423   private ZooKeeperWatcher zooKeeper;
424 
425   // master address manager and watcher
426   private MasterAddressTracker masterAddressManager;
427 
428   // Cluster Status Tracker
429   private ClusterStatusTracker clusterStatusTracker;
430 
431   // Log Splitting Worker
432   private SplitLogWorker splitLogWorker;
433 
434   // A sleeper that sleeps for msgInterval.
435   private final Sleeper sleeper;
436 
437   private final int rpcTimeout;
438 
439   private final RegionServerAccounting regionServerAccounting;
440 
441   // Cache configuration and block cache reference
442   final CacheConfig cacheConfig;
443 
444   /** The health check chore. */
445   private HealthCheckChore healthCheckChore;
446 
447   /** The nonce manager chore. */
448   private Chore nonceManagerChore;
449 
450   /**
451    * The server name the Master sees us as.  Its made from the hostname the
452    * master passes us, port, and server startcode. Gets set after registration
453    * against  Master.  The hostname can differ from the hostname in {@link #isa}
454    * but usually doesn't if both servers resolve .
455    */
456   private ServerName serverNameFromMasterPOV;
457 
458   /**
459    * This servers startcode.
460    */
461   private final long startcode;
462 
463   /**
464    * Unique identifier for the cluster we are a part of.
465    */
466   private String clusterId;
467 
468   /**
469    * MX Bean for RegionServerInfo
470    */
471   private ObjectName mxBean = null;
472 
473   /**
474    * Chore to clean periodically the moved region list
475    */
476   private MovedRegionsCleaner movedRegionsCleaner;
477 
478   /**
479    * The lease timeout period for client scanners (milliseconds).
480    */
481   private final int scannerLeaseTimeoutPeriod;
482 
483   /**
484    * The reference to the priority extraction function
485    */
486   private final PriorityFunction priority;
487 
488   private RegionServerCoprocessorHost rsHost;
489 
490   /** Handle all the snapshot requests to this server */
491   RegionServerSnapshotManager snapshotManager;
492 
493   // configuration setting on if replay WAL edits directly to another RS
494   private final boolean distributedLogReplay;
495 
496   // Table level lock manager for locking for region operations
497   private TableLockManager tableLockManager;
498 
499   /**
500    * Nonce manager. Nonces are used to make operations like increment and append idempotent
501    * in the case where client doesn't receive the response from a successful operation and
502    * retries. We track the successful ops for some time via a nonce sent by client and handle
503    * duplicate operations (currently, by failing them; in future we might use MVCC to return
504    * result). Nonces are also recovered from WAL during, recovery; however, the caveats (from
505    * HBASE-3787) are:
506    * - WAL recovery is optimized, and under high load we won't read nearly nonce-timeout worth
507    *   of past records. If we don't read the records, we don't read and recover the nonces.
508    *   Some WALs within nonce-timeout at recovery may not even be present due to rolling/cleanup.
509    * - There's no WAL recovery during normal region move, so nonces will not be transfered.
510    * We can have separate additional "Nonce WAL". It will just contain bunch of numbers and
511    * won't be flushed on main path - because WAL itself also contains nonces, if we only flush
512    * it before memstore flush, for a given nonce we will either see it in the WAL (if it was
513    * never flushed to disk, it will be part of recovery), or we'll see it as part of the nonce
514    * log (or both occasionally, which doesn't matter). Nonce log file can be deleted after the
515    * latest nonce in it expired. It can also be recovered during move.
516    */
517   private final ServerNonceManager nonceManager;
518 
519   private UserProvider userProvider;
520 
521   /**
522    * Starts a HRegionServer at the default location
523    *
524    * @param conf
525    * @throws IOException
526    * @throws InterruptedException
527    */
528   public HRegionServer(Configuration conf)
529   throws IOException, InterruptedException {
530     this.fsOk = true;
531     this.conf = conf;
532     this.isOnline = false;
533     checkCodecs(this.conf);
534     this.userProvider = UserProvider.instantiate(conf);
535 
536     FSUtils.setupShortCircuitRead(this.conf);
537 
538     // Config'ed params
539     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
540         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
541     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
542     this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
543 
544     this.sleeper = new Sleeper(this.msgInterval, this);
545 
546     boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true);
547     this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null;
548 
549     this.maxScannerResultSize = conf.getLong(
550       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
551       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
552 
553     this.numRegionsToReport = conf.getInt(
554       "hbase.regionserver.numregionstoreport", 10);
555 
556     this.rpcTimeout = conf.getInt(
557       HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY,
558       HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT);
559 
560     this.abortRequested = false;
561     this.stopped = false;
562 
563     this.scannerLeaseTimeoutPeriod = HBaseConfiguration.getInt(conf,
564       HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
565       HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
566       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
567 
568     // Server to handle client requests.
569     String hostname = conf.get("hbase.regionserver.ipc.address",
570       Strings.domainNamePointerToHostName(DNS.getDefaultHost(
571         conf.get("hbase.regionserver.dns.interface", "default"),
572         conf.get("hbase.regionserver.dns.nameserver", "default"))));
573     int port = conf.getInt(HConstants.REGIONSERVER_PORT,
574       HConstants.DEFAULT_REGIONSERVER_PORT);
575     // Creation of a HSA will force a resolve.
576     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
577     if (initialIsa.getAddress() == null) {
578       throw new IllegalArgumentException("Failed resolve of " + initialIsa);
579     }
580     this.rand = new Random(initialIsa.hashCode());
581     String name = "regionserver/" + initialIsa.toString();
582     // Set how many times to retry talking to another server over HConnection.
583     HConnectionManager.setServerSideHConnectionRetries(this.conf, name, LOG);
584     this.priority = new AnnotationReadingPriorityFunction(this);
585     RpcSchedulerFactory rpcSchedulerFactory;
586     try {
587       Class<?> rpcSchedulerFactoryClass = conf.getClass(
588           REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
589           SimpleRpcSchedulerFactory.class);
590       rpcSchedulerFactory = ((RpcSchedulerFactory) rpcSchedulerFactoryClass.newInstance());
591     } catch (InstantiationException e) {
592       throw new IllegalArgumentException(e);
593     } catch (IllegalAccessException e) {
594       throw new IllegalArgumentException(e);
595     }
596     this.rpcServer = new RpcServer(this, name, getServices(),
597       /*HBaseRPCErrorHandler.class, OnlineRegions.class},*/
598       initialIsa, // BindAddress is IP we got for this server.
599       conf,
600       rpcSchedulerFactory.create(conf, this));
601 
602     // Set our address.
603     this.isa = this.rpcServer.getListenerAddress();
604 
605     this.rpcServer.setErrorHandler(this);
606     this.startcode = System.currentTimeMillis();
607 
608     // login the zookeeper client principal (if using security)
609     ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
610       "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
611 
612     // login the server principal (if using secure Hadoop)
613     userProvider.login("hbase.regionserver.keytab.file",
614       "hbase.regionserver.kerberos.principal", this.isa.getHostName());
615     regionServerAccounting = new RegionServerAccounting();
616     cacheConfig = new CacheConfig(conf);
617     uncaughtExceptionHandler = new UncaughtExceptionHandler() {
618       @Override
619       public void uncaughtException(Thread t, Throwable e) {
620         abort("Uncaught exception in service thread " + t.getName(), e);
621       }
622     };
623 
624     this.rsInfo = RegionServerInfo.newBuilder();
625     // Put up the webui. Webui may come up on port other than configured if
626     // that port is occupied. Adjust serverInfo if this is the case.
627     this.rsInfo.setInfoPort(putUpWebUI());
628     this.distributedLogReplay = HLogSplitter.isDistributedLogReplay(this.conf);
629   }
630 
631   /**
632    * @return list of blocking services and their security info classes that this server supports
633    */
634   private List<BlockingServiceAndInterface> getServices() {
635     List<BlockingServiceAndInterface> bssi = new ArrayList<BlockingServiceAndInterface>(2);
636     bssi.add(new BlockingServiceAndInterface(
637         ClientProtos.ClientService.newReflectiveBlockingService(this),
638         ClientProtos.ClientService.BlockingInterface.class));
639     bssi.add(new BlockingServiceAndInterface(
640         AdminProtos.AdminService.newReflectiveBlockingService(this),
641         AdminProtos.AdminService.BlockingInterface.class));
642     return bssi;
643   }
644 
645   /**
646    * Run test on configured codecs to make sure supporting libs are in place.
647    * @param c
648    * @throws IOException
649    */
650   private static void checkCodecs(final Configuration c) throws IOException {
651     // check to see if the codec list is available:
652     String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null);
653     if (codecs == null) return;
654     for (String codec : codecs) {
655       if (!CompressionTest.testCompression(codec)) {
656         throw new IOException("Compression codec " + codec +
657           " not supported, aborting RS construction");
658       }
659     }
660   }
661 
662   String getClusterId() {
663     return this.clusterId;
664   }
665 
666   @Override
667   public int getPriority(RequestHeader header, Message param) {
668     return priority.getPriority(header, param);
669   }
670 
671   @Retention(RetentionPolicy.RUNTIME)
672   protected @interface QosPriority {
673     int priority() default 0;
674   }
675 
676   PriorityFunction getPriority() {
677     return priority;
678   }
679 
680   RegionScanner getScanner(long scannerId) {
681     String scannerIdString = Long.toString(scannerId);
682     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
683     if (scannerHolder != null) {
684       return scannerHolder.s;
685     }
686     return null;
687   }
688 
689   /**
690    * All initialization needed before we go register with Master.
691    *
692    * @throws IOException
693    * @throws InterruptedException
694    */
695   private void preRegistrationInitialization(){
696     try {
697       initializeZooKeeper();
698       initializeThreads();
699     } catch (Throwable t) {
700       // Call stop if error or process will stick around for ever since server
701       // puts up non-daemon threads.
702       this.rpcServer.stop();
703       abort("Initialization of RS failed.  Hence aborting RS.", t);
704     }
705   }
706 
707   /**
708    * Bring up connection to zk ensemble and then wait until a master for this
709    * cluster and then after that, wait until cluster 'up' flag has been set.
710    * This is the order in which master does things.
711    * Finally put up a catalog tracker.
712    * @throws IOException
713    * @throws InterruptedException
714    */
715   private void initializeZooKeeper() throws IOException, InterruptedException {
716     // Open connection to zookeeper and set primary watcher
717     this.zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +
718       this.isa.getPort(), this);
719 
720     // Create the master address manager, register with zk, and start it.  Then
721     // block until a master is available.  No point in starting up if no master
722     // running.
723     this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);
724     this.masterAddressManager.start();
725     blockAndCheckIfStopped(this.masterAddressManager);
726 
727     // Wait on cluster being up.  Master will set this flag up in zookeeper
728     // when ready.
729     this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
730     this.clusterStatusTracker.start();
731     blockAndCheckIfStopped(this.clusterStatusTracker);
732 
733     // Create the catalog tracker and start it;
734     this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
735     catalogTracker.start();
736 
737     // Retrieve clusterId
738     // Since cluster status is now up
739     // ID should have already been set by HMaster
740     try {
741       clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
742       if (clusterId == null) {
743         this.abort("Cluster ID has not been set");
744       }
745       LOG.info("ClusterId : "+clusterId);
746     } catch (KeeperException e) {
747       this.abort("Failed to retrieve Cluster ID",e);
748     }
749 
750     // watch for snapshots
751     try {
752       this.snapshotManager = new RegionServerSnapshotManager(this);
753     } catch (KeeperException e) {
754       this.abort("Failed to reach zk cluster when creating snapshot handler.");
755     }
756     this.tableLockManager = TableLockManager.createTableLockManager(conf, zooKeeper,
757         ServerName.valueOf(isa.getHostName(), isa.getPort(), startcode));
758 
759     // register watcher for recovering regions
760     if(this.distributedLogReplay) {
761       this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
762     }
763   }
764 
765   /**
766    * Utilty method to wait indefinitely on a znode availability while checking
767    * if the region server is shut down
768    * @param tracker znode tracker to use
769    * @throws IOException any IO exception, plus if the RS is stopped
770    * @throws InterruptedException
771    */
772   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
773       throws IOException, InterruptedException {
774     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
775       if (this.stopped) {
776         throw new IOException("Received the shutdown message while waiting.");
777       }
778     }
779   }
780 
781   /**
782    * @return False if cluster shutdown in progress
783    */
784   private boolean isClusterUp() {
785     return this.clusterStatusTracker.isClusterUp();
786   }
787 
788   private void initializeThreads() throws IOException {
789     // Cache flushing thread.
790     this.cacheFlusher = new MemStoreFlusher(conf, this);
791 
792     // Compaction thread
793     this.compactSplitThread = new CompactSplitThread(this);
794 
795     // Background thread to check for compactions; needed if region has not gotten updates
796     // in a while. It will take care of not checking too frequently on store-by-store basis.
797     this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
798     this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
799     // Health checker thread.
800     int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
801       HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
802     if (isHealthCheckerConfigured()) {
803       healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
804     }
805 
806     this.leases = new Leases(this.threadWakeFrequency);
807 
808     // Create the thread to clean the moved regions list
809     movedRegionsCleaner = MovedRegionsCleaner.createAndStart(this);
810 
811     if (this.nonceManager != null) {
812       // Create the chore that cleans up nonces.
813       nonceManagerChore = this.nonceManager.createCleanupChore(this);
814     }
815 
816     // Setup RPC client for master communication
817     rpcClient = new RpcClient(conf, clusterId, new InetSocketAddress(
818         this.isa.getAddress(), 0));
819     this.pauseMonitor = new JvmPauseMonitor(conf);
820     pauseMonitor.start();
821   }
822 
823   /**
824    * The HRegionServer sticks in this loop until closed.
825    */
826   @Override
827   public void run() {
828     try {
829       // Do pre-registration initializations; zookeeper, lease threads, etc.
830       preRegistrationInitialization();
831     } catch (Throwable e) {
832       abort("Fatal exception during initialization", e);
833     }
834 
835     try {
836       // Try and register with the Master; tell it we are here.  Break if
837       // server is stopped or the clusterup flag is down or hdfs went wacky.
838       while (keepLooping()) {
839         RegionServerStartupResponse w = reportForDuty();
840         if (w == null) {
841           LOG.warn("reportForDuty failed; sleeping and then retrying.");
842           this.sleeper.sleep();
843         } else {
844           handleReportForDutyResponse(w);
845           break;
846         }
847       }
848 
849       // Initialize the RegionServerCoprocessorHost now that our ephemeral
850       // node was created by reportForDuty, in case any coprocessors want
851       // to use ZooKeeper
852       this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
853 
854       if (!this.stopped && isHealthy()){
855         // start the snapshot handler, since the server is ready to run
856         this.snapshotManager.start();
857       }
858 
859       // We registered with the Master.  Go into run mode.
860       long lastMsg = 0;
861       long oldRequestCount = -1;
862       // The main run loop.
863       while (!this.stopped && isHealthy()) {
864         if (!isClusterUp()) {
865           if (isOnlineRegionsEmpty()) {
866             stop("Exiting; cluster shutdown set and not carrying any regions");
867           } else if (!this.stopping) {
868             this.stopping = true;
869             LOG.info("Closing user regions");
870             closeUserRegions(this.abortRequested);
871           } else if (this.stopping) {
872             boolean allUserRegionsOffline = areAllUserRegionsOffline();
873             if (allUserRegionsOffline) {
874               // Set stopped if no requests since last time we went around the loop.
875               // The remaining meta regions will be closed on our way out.
876               if (oldRequestCount == this.requestCount.get()) {
877                 stop("Stopped; only catalog regions remaining online");
878                 break;
879               }
880               oldRequestCount = this.requestCount.get();
881             } else {
882               // Make sure all regions have been closed -- some regions may
883               // have not got it because we were splitting at the time of
884               // the call to closeUserRegions.
885               closeUserRegions(this.abortRequested);
886             }
887             LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());
888           }
889         }
890         long now = System.currentTimeMillis();
891         if ((now - lastMsg) >= msgInterval) {
892           tryRegionServerReport(lastMsg, now);
893           lastMsg = System.currentTimeMillis();
894         }
895         if (!this.stopped) this.sleeper.sleep();
896       } // for
897     } catch (Throwable t) {
898       if (!checkOOME(t)) {
899         String prefix = t instanceof YouAreDeadException? "": "Unhandled: ";
900         abort(prefix + t.getMessage(), t);
901       }
902     }
903     // Run shutdown.
904     if (mxBean != null) {
905       MBeanUtil.unregisterMBean(mxBean);
906       mxBean = null;
907     }
908     if (this.leases != null) this.leases.closeAfterLeasesExpire();
909     this.rpcServer.stop();
910     if (this.splitLogWorker != null) {
911       splitLogWorker.stop();
912     }
913     if (this.infoServer != null) {
914       LOG.info("Stopping infoServer");
915       try {
916         this.infoServer.stop();
917       } catch (Exception e) {
918         e.printStackTrace();
919       }
920     }
921     // Send cache a shutdown.
922     if (cacheConfig.isBlockCacheEnabled()) {
923       cacheConfig.getBlockCache().shutdown();
924     }
925 
926     movedRegionsCleaner.stop("Region Server stopping");
927 
928     // Send interrupts to wake up threads if sleeping so they notice shutdown.
929     // TODO: Should we check they are alive? If OOME could have exited already
930     if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
931     if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
932     if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
933     if (this.metaHLogRoller != null) this.metaHLogRoller.interruptIfNecessary();
934     if (this.compactionChecker != null)
935       this.compactionChecker.interrupt();
936     if (this.healthCheckChore != null) {
937       this.healthCheckChore.interrupt();
938     }
939     if (this.nonceManagerChore != null) {
940       this.nonceManagerChore.interrupt();
941     }
942 
943     // Stop the snapshot handler, forcefully killing all running tasks
944     try {
945       if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
946     } catch (IOException e) {
947       LOG.warn("Failed to close snapshot handler cleanly", e);
948     }
949 
950     if (this.killed) {
951       // Just skip out w/o closing regions.  Used when testing.
952     } else if (abortRequested) {
953       if (this.fsOk) {
954         closeUserRegions(abortRequested); // Don't leave any open file handles
955       }
956       LOG.info("aborting server " + this.serverNameFromMasterPOV);
957     } else {
958       closeUserRegions(abortRequested);
959       closeAllScanners();
960       LOG.info("stopping server " + this.serverNameFromMasterPOV);
961     }
962     // Interrupt catalog tracker here in case any regions being opened out in
963     // handlers are stuck waiting on meta.
964     if (this.catalogTracker != null) this.catalogTracker.stop();
965 
966     // Closing the compactSplit thread before closing meta regions
967     if (!this.killed && containsMetaTableRegions()) {
968       if (!abortRequested || this.fsOk) {
969         if (this.compactSplitThread != null) {
970           this.compactSplitThread.join();
971           this.compactSplitThread = null;
972         }
973         closeMetaTableRegions(abortRequested);
974       }
975     }
976 
977     if (!this.killed && this.fsOk) {
978       waitOnAllRegionsToClose(abortRequested);
979       LOG.info("stopping server " + this.serverNameFromMasterPOV +
980         "; all regions closed.");
981     }
982 
983     //fsOk flag may be changed when closing regions throws exception.
984     if (this.fsOk) {
985       closeWAL(!abortRequested);
986     }
987 
988     // Make sure the proxy is down.
989     if (this.rssStub != null) {
990       this.rssStub = null;
991     }
992     this.rpcClient.stop();
993     this.leases.close();
994     if (this.pauseMonitor != null) {
995       this.pauseMonitor.stop();
996     }
997 
998     if (!killed) {
999       join();
1000     }
1001 
1002     try {
1003       deleteMyEphemeralNode();
1004     } catch (KeeperException e) {
1005       LOG.warn("Failed deleting my ephemeral node", e);
1006     }
1007     // We may have failed to delete the znode at the previous step, but
1008     //  we delete the file anyway: a second attempt to delete the znode is likely to fail again.
1009     ZNodeClearer.deleteMyEphemeralNodeOnDisk();
1010     this.zooKeeper.close();
1011     LOG.info("stopping server " + this.serverNameFromMasterPOV +
1012       "; zookeeper connection closed.");
1013 
1014     LOG.info(Thread.currentThread().getName() + " exiting");
1015   }
1016 
1017   private boolean containsMetaTableRegions() {
1018     return onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1019   }
1020 
1021   private boolean areAllUserRegionsOffline() {
1022     if (getNumberOfOnlineRegions() > 2) return false;
1023     boolean allUserRegionsOffline = true;
1024     for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
1025       if (!e.getValue().getRegionInfo().isMetaTable()) {
1026         allUserRegionsOffline = false;
1027         break;
1028       }
1029     }
1030     return allUserRegionsOffline;
1031   }
1032 
1033   void tryRegionServerReport(long reportStartTime, long reportEndTime)
1034   throws IOException {
1035     if (this.rssStub == null) {
1036       // the current server is stopping.
1037       return;
1038     }
1039     ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
1040     try {
1041       RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();
1042       ServerName sn = ServerName.parseVersionedServerName(
1043         this.serverNameFromMasterPOV.getVersionedBytes());
1044       request.setServer(ProtobufUtil.toServerName(sn));
1045       request.setLoad(sl);
1046       this.rssStub.regionServerReport(null, request.build());
1047     } catch (ServiceException se) {
1048       IOException ioe = ProtobufUtil.getRemoteException(se);
1049       if (ioe instanceof YouAreDeadException) {
1050         // This will be caught and handled as a fatal error in run()
1051         throw ioe;
1052       }
1053       // Couldn't connect to the master, get location from zk and reconnect
1054       // Method blocks until new master is found or we are stopped
1055       Pair<ServerName, RegionServerStatusService.BlockingInterface> p =
1056         createRegionServerStatusStub();
1057       this.rssStub = p.getSecond();
1058     }
1059   }
1060 
1061   ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime) {
1062     // We're getting the MetricsRegionServerWrapper here because the wrapper computes requests
1063     // per second, and other metrics  As long as metrics are part of ServerLoad it's best to use
1064     // the wrapper to compute those numbers in one place.
1065     // In the long term most of these should be moved off of ServerLoad and the heart beat.
1066     // Instead they should be stored in an HBase table so that external visibility into HBase is
1067     // improved; Additionally the load balancer will be able to take advantage of a more complete
1068     // history.
1069     MetricsRegionServerWrapper regionServerWrapper = this.metricsRegionServer.getRegionServerWrapper();
1070     Collection<HRegion> regions = getOnlineRegionsLocalContext();
1071     MemoryUsage memory =
1072       ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
1073 
1074     ClusterStatusProtos.ServerLoad.Builder serverLoad =
1075       ClusterStatusProtos.ServerLoad.newBuilder();
1076     serverLoad.setNumberOfRequests((int) regionServerWrapper.getRequestsPerSecond());
1077     serverLoad.setTotalNumberOfRequests((int) regionServerWrapper.getTotalRequestCount());
1078     serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024));
1079     serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024));
1080     Set<String> coprocessors = this.hlog.getCoprocessorHost().getCoprocessors();
1081     for (String coprocessor : coprocessors) {
1082       serverLoad.addCoprocessors(
1083         Coprocessor.newBuilder().setName(coprocessor).build());
1084     }
1085     for (HRegion region : regions) {
1086       serverLoad.addRegionLoads(createRegionLoad(region));
1087     }
1088     serverLoad.setReportStartTime(reportStartTime);
1089     serverLoad.setReportEndTime(reportEndTime);
1090     if (this.infoServer != null) {
1091       serverLoad.setInfoServerPort(this.infoServer.getPort());
1092     } else {
1093       serverLoad.setInfoServerPort(-1);
1094     }
1095     return serverLoad.build();
1096   }
1097 
1098   String getOnlineRegionsAsPrintableString() {
1099     StringBuilder sb = new StringBuilder();
1100     for (HRegion r: this.onlineRegions.values()) {
1101       if (sb.length() > 0) sb.append(", ");
1102       sb.append(r.getRegionInfo().getEncodedName());
1103     }
1104     return sb.toString();
1105   }
1106 
1107   /**
1108    * Wait on regions close.
1109    */
1110   private void waitOnAllRegionsToClose(final boolean abort) {
1111     // Wait till all regions are closed before going out.
1112     int lastCount = -1;
1113     long previousLogTime = 0;
1114     Set<String> closedRegions = new HashSet<String>();
1115     while (!isOnlineRegionsEmpty()) {
1116       int count = getNumberOfOnlineRegions();
1117       // Only print a message if the count of regions has changed.
1118       if (count != lastCount) {
1119         // Log every second at most
1120         if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1121           previousLogTime = System.currentTimeMillis();
1122           lastCount = count;
1123           LOG.info("Waiting on " + count + " regions to close");
1124           // Only print out regions still closing if a small number else will
1125           // swamp the log.
1126           if (count < 10 && LOG.isDebugEnabled()) {
1127             LOG.debug(this.onlineRegions);
1128           }
1129         }
1130       }
1131       // Ensure all user regions have been sent a close. Use this to
1132       // protect against the case where an open comes in after we start the
1133       // iterator of onlineRegions to close all user regions.
1134       for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
1135         HRegionInfo hri = e.getValue().getRegionInfo();
1136         if (!this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes())
1137             && !closedRegions.contains(hri.getEncodedName())) {
1138           closedRegions.add(hri.getEncodedName());
1139           // Don't update zk with this close transition; pass false.
1140           closeRegionIgnoreErrors(hri, abort);
1141         }
1142       }
1143       // No regions in RIT, we could stop waiting now.
1144       if (this.regionsInTransitionInRS.isEmpty()) {
1145         if (!isOnlineRegionsEmpty()) {
1146           LOG.info("We were exiting though online regions are not empty," +
1147               " because some regions failed closing");
1148         }
1149         break;
1150       }
1151       Threads.sleep(200);
1152     }
1153   }
1154 
1155   private void closeWAL(final boolean delete) {
1156     if (this.hlogForMeta != null) {
1157       // All hlogs (meta and non-meta) are in the same directory. Don't call
1158       // closeAndDelete here since that would delete all hlogs not just the
1159       // meta ones. We will just 'close' the hlog for meta here, and leave
1160       // the directory cleanup to the follow-on closeAndDelete call.
1161       try {
1162         this.hlogForMeta.close();
1163       } catch (Throwable e) {
1164         LOG.error("Metalog close and delete failed", RemoteExceptionHandler.checkThrowable(e));
1165       }
1166     }
1167     if (this.hlog != null) {
1168       try {
1169         if (delete) {
1170           hlog.closeAndDelete();
1171         } else {
1172           hlog.close();
1173         }
1174       } catch (Throwable e) {
1175         LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
1176       }
1177     }
1178   }
1179 
1180   private void closeAllScanners() {
1181     // Close any outstanding scanners. Means they'll get an UnknownScanner
1182     // exception next time they come in.
1183     for (Map.Entry<String, RegionScannerHolder> e : this.scanners.entrySet()) {
1184       try {
1185         e.getValue().s.close();
1186       } catch (IOException ioe) {
1187         LOG.warn("Closing scanner " + e.getKey(), ioe);
1188       }
1189     }
1190   }
1191 
1192   /*
1193    * Run init. Sets up hlog and starts up all server threads.
1194    *
1195    * @param c Extra configuration.
1196    */
1197   protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
1198   throws IOException {
1199     try {
1200       for (NameStringPair e : c.getMapEntriesList()) {
1201         String key = e.getName();
1202         // The hostname the master sees us as.
1203         if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
1204           String hostnameFromMasterPOV = e.getValue();
1205           this.serverNameFromMasterPOV = ServerName.valueOf(hostnameFromMasterPOV,
1206               this.isa.getPort(), this.startcode);
1207           if (!hostnameFromMasterPOV.equals(this.isa.getHostName())) {
1208             LOG.info("Master passed us a different hostname to use; was=" +
1209               this.isa.getHostName() + ", but now=" + hostnameFromMasterPOV);
1210           }
1211           continue;
1212         }
1213         String value = e.getValue();
1214         if (LOG.isDebugEnabled()) {
1215           LOG.debug("Config from master: " + key + "=" + value);
1216         }
1217         this.conf.set(key, value);
1218       }
1219 
1220       // hack! Maps DFSClient => RegionServer for logs.  HDFS made this
1221       // config param for task trackers, but we can piggyback off of it.
1222       if (this.conf.get("mapred.task.id") == null) {
1223         this.conf.set("mapred.task.id", "hb_rs_" +
1224           this.serverNameFromMasterPOV.toString());
1225       }
1226       // Set our ephemeral znode up in zookeeper now we have a name.
1227       createMyEphemeralNode();
1228 
1229       // Save it in a file, this will allow to see if we crash
1230       ZNodeClearer.writeMyEphemeralNodeOnDisk(getMyEphemeralNodePath());
1231 
1232       // Master sent us hbase.rootdir to use. Should be fully qualified
1233       // path with file system specification included. Set 'fs.defaultFS'
1234       // to match the filesystem on hbase.rootdir else underlying hadoop hdfs
1235       // accessors will be going against wrong filesystem (unless all is set
1236       // to defaults).
1237       FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf));
1238       // Get fs instance used by this RS.  Do we use checksum verification in the hbase? If hbase
1239       // checksum verification enabled, then automatically switch off hdfs checksum verification.
1240       boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1241       this.fs = new HFileSystem(this.conf, useHBaseChecksum);
1242       this.rootDir = FSUtils.getRootDir(this.conf);
1243       this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
1244       this.hlog = setupWALAndReplication();
1245       // Init in here rather than in constructor after thread name has been set
1246       this.metricsRegionServer = new MetricsRegionServer(new MetricsRegionServerWrapperImpl(this));
1247 
1248       spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
1249 
1250       startServiceThreads();
1251       LOG.info("Serving as " + this.serverNameFromMasterPOV +
1252         ", RpcServer on " + this.isa +
1253         ", sessionid=0x" +
1254         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
1255       isOnline = true;
1256     } catch (Throwable e) {
1257       this.isOnline = false;
1258       stop("Failed initialization");
1259       throw convertThrowableToIOE(cleanup(e, "Failed init"),
1260           "Region server startup failed");
1261     } finally {
1262       sleeper.skipSleepCycle();
1263     }
1264   }
1265 
1266   private void createMyEphemeralNode() throws KeeperException, IOException {
1267     byte[] data = ProtobufUtil.prependPBMagic(rsInfo.build().toByteArray());
1268     ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper,
1269       getMyEphemeralNodePath(), data);
1270   }
1271 
1272   private void deleteMyEphemeralNode() throws KeeperException {
1273     ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
1274   }
1275 
1276   @Override
1277   public RegionServerAccounting getRegionServerAccounting() {
1278     return regionServerAccounting;
1279   }
1280 
1281   @Override
1282   public TableLockManager getTableLockManager() {
1283     return tableLockManager;
1284   }
1285 
1286   /*
1287    * @param r Region to get RegionLoad for.
1288    *
1289    * @return RegionLoad instance.
1290    *
1291    * @throws IOException
1292    */
1293   private RegionLoad createRegionLoad(final HRegion r) {
1294     byte[] name = r.getRegionName();
1295     int stores = 0;
1296     int storefiles = 0;
1297     int storeUncompressedSizeMB = 0;
1298     int storefileSizeMB = 0;
1299     int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
1300     int storefileIndexSizeMB = 0;
1301     int rootIndexSizeKB = 0;
1302     int totalStaticIndexSizeKB = 0;
1303     int totalStaticBloomSizeKB = 0;
1304     long totalCompactingKVs = 0;
1305     long currentCompactedKVs = 0;
1306     synchronized (r.stores) {
1307       stores += r.stores.size();
1308       for (Store store : r.stores.values()) {
1309         storefiles += store.getStorefilesCount();
1310         storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed()
1311             / 1024 / 1024);
1312         storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
1313         storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
1314         CompactionProgress progress = store.getCompactionProgress();
1315         if (progress != null) {
1316           totalCompactingKVs += progress.totalCompactingKVs;
1317           currentCompactedKVs += progress.currentCompactedKVs;
1318         }
1319 
1320         rootIndexSizeKB +=
1321             (int) (store.getStorefilesIndexSize() / 1024);
1322 
1323         totalStaticIndexSizeKB +=
1324           (int) (store.getTotalStaticIndexSize() / 1024);
1325 
1326         totalStaticBloomSizeKB +=
1327           (int) (store.getTotalStaticBloomSize() / 1024);
1328       }
1329     }
1330     RegionLoad.Builder regionLoad = RegionLoad.newBuilder();
1331     RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder();
1332     regionSpecifier.setType(RegionSpecifierType.REGION_NAME);
1333     regionSpecifier.setValue(HBaseZeroCopyByteString.wrap(name));
1334     regionLoad.setRegionSpecifier(regionSpecifier.build())
1335       .setStores(stores)
1336       .setStorefiles(storefiles)
1337       .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
1338       .setStorefileSizeMB(storefileSizeMB)
1339       .setMemstoreSizeMB(memstoreSizeMB)
1340       .setStorefileIndexSizeMB(storefileIndexSizeMB)
1341       .setRootIndexSizeKB(rootIndexSizeKB)
1342       .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB)
1343       .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB)
1344       .setReadRequestsCount((int) r.readRequestsCount.get())
1345       .setWriteRequestsCount((int) r.writeRequestsCount.get())
1346       .setTotalCompactingKVs(totalCompactingKVs)
1347       .setCurrentCompactedKVs(currentCompactedKVs)
1348       .setCompleteSequenceId(r.completeSequenceId);
1349 
1350     return regionLoad.build();
1351   }
1352 
1353   /**
1354    * @param encodedRegionName
1355    * @return An instance of RegionLoad.
1356    */
1357   public RegionLoad createRegionLoad(final String encodedRegionName) {
1358     HRegion r = null;
1359     r = this.onlineRegions.get(encodedRegionName);
1360     return r != null ? createRegionLoad(r) : null;
1361   }
1362 
1363   /*
1364    * Inner class that runs on a long period checking if regions need compaction.
1365    */
1366   private static class CompactionChecker extends Chore {
1367     private final HRegionServer instance;
1368     private final int majorCompactPriority;
1369     private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE;
1370     private long iteration = 0;
1371 
1372     CompactionChecker(final HRegionServer h, final int sleepTime,
1373         final Stoppable stopper) {
1374       super("CompactionChecker", sleepTime, h);
1375       this.instance = h;
1376       LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime));
1377 
1378       /* MajorCompactPriority is configurable.
1379        * If not set, the compaction will use default priority.
1380        */
1381       this.majorCompactPriority = this.instance.conf.
1382         getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
1383         DEFAULT_PRIORITY);
1384     }
1385 
1386     @Override
1387     protected void chore() {
1388       for (HRegion r : this.instance.onlineRegions.values()) {
1389         if (r == null)
1390           continue;
1391         for (Store s : r.getStores().values()) {
1392           try {
1393             long multiplier = s.getCompactionCheckMultiplier();
1394             assert multiplier > 0;
1395             if (iteration % multiplier != 0) continue;
1396             if (s.needsCompaction()) {
1397               // Queue a compaction. Will recognize if major is needed.
1398               this.instance.compactSplitThread.requestSystemCompaction(r, s, getName()
1399                   + " requests compaction");
1400             } else if (s.isMajorCompaction()) {
1401               if (majorCompactPriority == DEFAULT_PRIORITY
1402                   || majorCompactPriority > r.getCompactPriority()) {
1403                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1404                     + " requests major compaction; use default priority", null);
1405               } else {
1406                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1407                     + " requests major compaction; use configured priority",
1408                   this.majorCompactPriority, null);
1409               }
1410             }
1411           } catch (IOException e) {
1412             LOG.warn("Failed major compaction check on " + r, e);
1413           }
1414         }
1415       }
1416       iteration = (iteration == Long.MAX_VALUE) ? 0 : (iteration + 1);
1417     }
1418   }
1419 
1420   class PeriodicMemstoreFlusher extends Chore {
1421     final HRegionServer server;
1422     final static int RANGE_OF_DELAY = 20000; //millisec
1423     final static int MIN_DELAY_TIME = 3000; //millisec
1424     public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
1425       super(server.getServerName() + "-MemstoreFlusherChore", cacheFlushInterval, server);
1426       this.server = server;
1427     }
1428 
1429     @Override
1430     protected void chore() {
1431       for (HRegion r : this.server.onlineRegions.values()) {
1432         if (r == null)
1433           continue;
1434         if (r.shouldFlush()) {
1435           FlushRequester requester = server.getFlushRequester();
1436           if (requester != null) {
1437             long randomDelay = rand.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
1438             LOG.info(getName() + " requesting flush for region " + r.getRegionNameAsString() +
1439                 " after a delay of " + randomDelay);
1440             //Throttle the flushes by putting a delay. If we don't throttle, and there
1441             //is a balanced write-load on the regions in a table, we might end up
1442             //overwhelming the filesystem with too many flushes at once.
1443             requester.requestDelayedFlush(r, randomDelay);
1444           }
1445         }
1446       }
1447     }
1448   }
1449 
1450   /**
1451    * Report the status of the server. A server is online once all the startup is
1452    * completed (setting up filesystem, starting service threads, etc.). This
1453    * method is designed mostly to be useful in tests.
1454    *
1455    * @return true if online, false if not.
1456    */
1457   public boolean isOnline() {
1458     return isOnline;
1459   }
1460 
1461   /**
1462    * Setup WAL log and replication if enabled.
1463    * Replication setup is done in here because it wants to be hooked up to WAL.
1464    * @return A WAL instance.
1465    * @throws IOException
1466    */
1467   private HLog setupWALAndReplication() throws IOException {
1468     final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
1469     final String logName
1470       = HLogUtil.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
1471 
1472     Path logdir = new Path(rootDir, logName);
1473     if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
1474     if (this.fs.exists(logdir)) {
1475       throw new RegionServerRunningException("Region server has already " +
1476         "created directory at " + this.serverNameFromMasterPOV.toString());
1477     }
1478 
1479     // Instantiate replication manager if replication enabled.  Pass it the
1480     // log directories.
1481     createNewReplicationInstance(conf, this, this.fs, logdir, oldLogDir);
1482 
1483     return instantiateHLog(rootDir, logName);
1484   }
1485 
1486   private HLog getMetaWAL() throws IOException {
1487     if (this.hlogForMeta != null) return this.hlogForMeta;
1488     final String logName = HLogUtil.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
1489     Path logdir = new Path(rootDir, logName);
1490     if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
1491     this.hlogForMeta = HLogFactory.createMetaHLog(this.fs.getBackingFs(), rootDir, logName,
1492       this.conf, getMetaWALActionListeners(), this.serverNameFromMasterPOV.toString());
1493     return this.hlogForMeta;
1494   }
1495 
1496   /**
1497    * Called by {@link #setupWALAndReplication()} creating WAL instance.
1498    * @param rootdir
1499    * @param logName
1500    * @return WAL instance.
1501    * @throws IOException
1502    */
1503   protected HLog instantiateHLog(Path rootdir, String logName) throws IOException {
1504     return HLogFactory.createHLog(this.fs.getBackingFs(), rootdir, logName, this.conf,
1505       getWALActionListeners(), this.serverNameFromMasterPOV.toString());
1506   }
1507 
1508   /**
1509    * Called by {@link #instantiateHLog(Path, String)} setting up WAL instance.
1510    * Add any {@link WALActionsListener}s you want inserted before WAL startup.
1511    * @return List of WALActionsListener that will be passed in to
1512    * {@link org.apache.hadoop.hbase.regionserver.wal.FSHLog} on construction.
1513    */
1514   protected List<WALActionsListener> getWALActionListeners() {
1515     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1516     // Log roller.
1517     this.hlogRoller = new LogRoller(this, this);
1518     listeners.add(this.hlogRoller);
1519     if (this.replicationSourceHandler != null &&
1520         this.replicationSourceHandler.getWALActionsListener() != null) {
1521       // Replication handler is an implementation of WALActionsListener.
1522       listeners.add(this.replicationSourceHandler.getWALActionsListener());
1523     }
1524     return listeners;
1525   }
1526 
1527   protected List<WALActionsListener> getMetaWALActionListeners() {
1528     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1529     // Using a tmp log roller to ensure metaLogRoller is alive once it is not
1530     // null
1531     MetaLogRoller tmpLogRoller = new MetaLogRoller(this, this);
1532     String n = Thread.currentThread().getName();
1533     Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
1534         n + "-MetaLogRoller", uncaughtExceptionHandler);
1535     this.metaHLogRoller = tmpLogRoller;
1536     tmpLogRoller = null;
1537     listeners.add(this.metaHLogRoller);
1538     return listeners;
1539   }
1540 
1541   protected LogRoller getLogRoller() {
1542     return hlogRoller;
1543   }
1544 
1545   public MetricsRegionServer getMetrics() {
1546     return this.metricsRegionServer;
1547   }
1548 
1549   /**
1550    * @return Master address tracker instance.
1551    */
1552   public MasterAddressTracker getMasterAddressManager() {
1553     return this.masterAddressManager;
1554   }
1555 
1556   /*
1557    * Start maintenance Threads, Server, Worker and lease checker threads.
1558    * Install an UncaughtExceptionHandler that calls abort of RegionServer if we
1559    * get an unhandled exception. We cannot set the handler on all threads.
1560    * Server's internal Listener thread is off limits. For Server, if an OOME, it
1561    * waits a while then retries. Meantime, a flush or a compaction that tries to
1562    * run should trigger same critical condition and the shutdown will run. On
1563    * its way out, this server will shut down Server. Leases are sort of
1564    * inbetween. It has an internal thread that while it inherits from Chore, it
1565    * keeps its own internal stop mechanism so needs to be stopped by this
1566    * hosting server. Worker logs the exception and exits.
1567    */
1568   private void startServiceThreads() throws IOException {
1569     String n = Thread.currentThread().getName();
1570     // Start executor services
1571     this.service = new ExecutorService(getServerName().toShortString());
1572     this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
1573       conf.getInt("hbase.regionserver.executor.openregion.threads", 3));
1574     this.service.startExecutorService(ExecutorType.RS_OPEN_META,
1575       conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
1576     this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
1577       conf.getInt("hbase.regionserver.executor.closeregion.threads", 3));
1578     this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
1579       conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
1580     if (conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false)) {
1581       this.service.startExecutorService(ExecutorType.RS_PARALLEL_SEEK,
1582         conf.getInt("hbase.storescanner.parallel.seek.threads", 10));
1583     }
1584     this.service.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS,
1585       conf.getInt("hbase.regionserver.wal.max.splitters", SplitLogWorker.DEFAULT_MAX_SPLITTERS));
1586 
1587     Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller",
1588         uncaughtExceptionHandler);
1589     this.cacheFlusher.start(uncaughtExceptionHandler);
1590     Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
1591       ".compactionChecker", uncaughtExceptionHandler);
1592     Threads.setDaemonThreadRunning(this.periodicFlusher.getThread(), n +
1593         ".periodicFlusher", uncaughtExceptionHandler);
1594     if (this.healthCheckChore != null) {
1595       Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
1596             uncaughtExceptionHandler);
1597     }
1598     if (this.nonceManagerChore != null) {
1599       Threads.setDaemonThreadRunning(this.nonceManagerChore.getThread(), n + ".nonceCleaner",
1600             uncaughtExceptionHandler);
1601     }
1602 
1603     // Leases is not a Thread. Internally it runs a daemon thread. If it gets
1604     // an unhandled exception, it will just exit.
1605     this.leases.setName(n + ".leaseChecker");
1606     this.leases.start();
1607 
1608     if (this.replicationSourceHandler == this.replicationSinkHandler &&
1609         this.replicationSourceHandler != null) {
1610       this.replicationSourceHandler.startReplicationService();
1611     } else {
1612       if (this.replicationSourceHandler != null) {
1613         this.replicationSourceHandler.startReplicationService();
1614       }
1615       if (this.replicationSinkHandler != null) {
1616         this.replicationSinkHandler.startReplicationService();
1617       }
1618     }
1619 
1620     // Start Server.  This service is like leases in that it internally runs
1621     // a thread.
1622     this.rpcServer.start();
1623 
1624     // Create the log splitting worker and start it
1625     // set a smaller retries to fast fail otherwise splitlogworker could be blocked for
1626     // quite a while inside HConnection layer. The worker won't be available for other
1627     // tasks even after current task is preempted after a split task times out.
1628     Configuration sinkConf = HBaseConfiguration.create(conf);
1629     sinkConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
1630       conf.getInt("hbase.log.replay.retries.number", 8)); // 8 retries take about 23 seconds
1631     sinkConf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1632       conf.getInt("hbase.log.replay.rpc.timeout", 30000)); // default 30 seconds
1633     sinkConf.setInt("hbase.client.serverside.retries.multiplier", 1);
1634     this.splitLogWorker = new SplitLogWorker(this.zooKeeper, sinkConf, this, this);
1635     splitLogWorker.start();
1636   }
1637 
1638   /**
1639    * Puts up the webui.
1640    * @return Returns final port -- maybe different from what we started with.
1641    * @throws IOException
1642    */
1643   private int putUpWebUI() throws IOException {
1644     int port = this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 60030);
1645     // -1 is for disabling info server
1646     if (port < 0) return port;
1647     String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
1648     // check if auto port bind enabled
1649     boolean auto = this.conf.getBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,
1650         false);
1651     while (true) {
1652       try {
1653         this.infoServer = new InfoServer("regionserver", addr, port, false, this.conf);
1654         this.infoServer.addServlet("status", "/rs-status", RSStatusServlet.class);
1655         this.infoServer.addServlet("dump", "/dump", RSDumpServlet.class);
1656         this.infoServer.setAttribute(REGIONSERVER, this);
1657         this.infoServer.setAttribute(REGIONSERVER_CONF, conf);
1658         this.infoServer.start();
1659         break;
1660       } catch (BindException e) {
1661         if (!auto) {
1662           // auto bind disabled throw BindException
1663           LOG.error("Failed binding http info server to port: " + port);
1664           throw e;
1665         }
1666         // auto bind enabled, try to use another port
1667         LOG.info("Failed binding http info server to port: " + port);
1668         port++;
1669       }
1670     }
1671     return this.infoServer.getPort();
1672   }
1673 
1674   /*
1675    * Verify that server is healthy
1676    */
1677   private boolean isHealthy() {
1678     if (!fsOk) {
1679       // File system problem
1680       return false;
1681     }
1682     // Verify that all threads are alive
1683     if (!(leases.isAlive()
1684         && cacheFlusher.isAlive() && hlogRoller.isAlive()
1685         && this.compactionChecker.isAlive())
1686         && this.periodicFlusher.isAlive()) {
1687       stop("One or more threads are no longer alive -- stop");
1688       return false;
1689     }
1690     if (metaHLogRoller != null && !metaHLogRoller.isAlive()) {
1691       stop("Meta HLog roller thread is no longer alive -- stop");
1692       return false;
1693     }
1694     return true;
1695   }
1696 
1697   public HLog getWAL() {
1698     try {
1699       return getWAL(null);
1700     } catch (IOException e) {
1701       LOG.warn("getWAL threw exception " + e);
1702       return null;
1703     }
1704   }
1705 
1706   @Override
1707   public HLog getWAL(HRegionInfo regionInfo) throws IOException {
1708     //TODO: at some point this should delegate to the HLogFactory
1709     //currently, we don't care about the region as much as we care about the
1710     //table.. (hence checking the tablename below)
1711     //_ROOT_ and hbase:meta regions have separate WAL.
1712     if (regionInfo != null && regionInfo.isMetaTable()) {
1713       return getMetaWAL();
1714     }
1715     return this.hlog;
1716   }
1717 
1718   @Override
1719   public CatalogTracker getCatalogTracker() {
1720     return this.catalogTracker;
1721   }
1722 
1723   @Override
1724   public void stop(final String msg) {
1725     try {
1726     	if (this.rsHost != null) {
1727     		this.rsHost.preStop(msg);
1728     	}
1729       this.stopped = true;
1730       LOG.info("STOPPED: " + msg);
1731       // Wakes run() if it is sleeping
1732       sleeper.skipSleepCycle();
1733     } catch (IOException exp) {
1734       LOG.warn("The region server did not stop", exp);
1735     }
1736   }
1737 
1738   public void waitForServerOnline(){
1739     while (!isOnline() && !isStopped()){
1740        sleeper.sleep();
1741     }
1742   }
1743 
1744   @Override
1745   public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct)
1746   throws KeeperException, IOException {
1747     checkOpen();
1748     LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString());
1749     // Do checks to see if we need to compact (references or too many files)
1750     for (Store s : r.getStores().values()) {
1751       if (s.hasReferences() || s.needsCompaction()) {
1752        this.compactSplitThread.requestSystemCompaction(r, s, "Opening Region");
1753       }
1754     }
1755     long openSeqNum = r.getOpenSeqNum();
1756     if (openSeqNum == HConstants.NO_SEQNUM) {
1757       // If we opened a region, we should have read some sequence number from it.
1758       LOG.error("No sequence number found when opening " + r.getRegionNameAsString());
1759       openSeqNum = 0;
1760     }
1761 
1762     // Update flushed sequence id of a recovering region in ZK
1763     updateRecoveringRegionLastFlushedSequenceId(r);
1764 
1765     // Update ZK, or META
1766     if (r.getRegionInfo().isMetaRegion()) {
1767       MetaRegionTracker.setMetaLocation(getZooKeeper(),
1768           this.serverNameFromMasterPOV);
1769     } else {
1770       MetaEditor.updateRegionLocation(ct, r.getRegionInfo(),
1771         this.serverNameFromMasterPOV, openSeqNum);
1772     }
1773     LOG.info("Finished post open deploy task for " + r.getRegionNameAsString());
1774 
1775   }
1776 
1777   @Override
1778   public RpcServerInterface getRpcServer() {
1779     return rpcServer;
1780   }
1781 
1782   /**
1783    * Cause the server to exit without closing the regions it is serving, the log
1784    * it is using and without notifying the master. Used unit testing and on
1785    * catastrophic events such as HDFS is yanked out from under hbase or we OOME.
1786    *
1787    * @param reason
1788    *          the reason we are aborting
1789    * @param cause
1790    *          the exception that caused the abort, or null
1791    */
1792   @Override
1793   public void abort(String reason, Throwable cause) {
1794     String msg = "ABORTING region server " + this + ": " + reason;
1795     if (cause != null) {
1796       LOG.fatal(msg, cause);
1797     } else {
1798       LOG.fatal(msg);
1799     }
1800     this.abortRequested = true;
1801     // HBASE-4014: show list of coprocessors that were loaded to help debug
1802     // regionserver crashes.Note that we're implicitly using
1803     // java.util.HashSet's toString() method to print the coprocessor names.
1804     LOG.fatal("RegionServer abort: loaded coprocessors are: " +
1805         CoprocessorHost.getLoadedCoprocessors());
1806     // Do our best to report our abort to the master, but this may not work
1807     try {
1808       if (cause != null) {
1809         msg += "\nCause:\n" + StringUtils.stringifyException(cause);
1810       }
1811       // Report to the master but only if we have already registered with the master.
1812       if (rssStub != null && this.serverNameFromMasterPOV != null) {
1813         ReportRSFatalErrorRequest.Builder builder =
1814           ReportRSFatalErrorRequest.newBuilder();
1815         ServerName sn =
1816           ServerName.parseVersionedServerName(this.serverNameFromMasterPOV.getVersionedBytes());
1817         builder.setServer(ProtobufUtil.toServerName(sn));
1818         builder.setErrorMessage(msg);
1819         rssStub.reportRSFatalError(null, builder.build());
1820       }
1821     } catch (Throwable t) {
1822       LOG.warn("Unable to report fatal error to master", t);
1823     }
1824     stop(reason);
1825   }
1826 
1827   /**
1828    * @see HRegionServer#abort(String, Throwable)
1829    */
1830   public void abort(String reason) {
1831     abort(reason, null);
1832   }
1833 
1834   @Override
1835   public boolean isAborted() {
1836     return this.abortRequested;
1837   }
1838 
1839   /*
1840    * Simulate a kill -9 of this server. Exits w/o closing regions or cleaninup
1841    * logs but it does close socket in case want to bring up server on old
1842    * hostname+port immediately.
1843    */
1844   protected void kill() {
1845     this.killed = true;
1846     abort("Simulated kill");
1847   }
1848 
1849   /**
1850    * Wait on all threads to finish. Presumption is that all closes and stops
1851    * have already been called.
1852    */
1853   protected void join() {
1854     if (this.nonceManagerChore != null) {
1855       Threads.shutdown(this.nonceManagerChore.getThread());
1856     }
1857     Threads.shutdown(this.compactionChecker.getThread());
1858     Threads.shutdown(this.periodicFlusher.getThread());
1859     this.cacheFlusher.join();
1860     if (this.healthCheckChore != null) {
1861       Threads.shutdown(this.healthCheckChore.getThread());
1862     }
1863     if (this.spanReceiverHost != null) {
1864       this.spanReceiverHost.closeReceivers();
1865     }
1866     if (this.hlogRoller != null) {
1867       Threads.shutdown(this.hlogRoller.getThread());
1868     }
1869     if (this.metaHLogRoller != null) {
1870       Threads.shutdown(this.metaHLogRoller.getThread());
1871     }
1872     if (this.compactSplitThread != null) {
1873       this.compactSplitThread.join();
1874     }
1875     if (this.service != null) this.service.shutdown();
1876     if (this.replicationSourceHandler != null &&
1877         this.replicationSourceHandler == this.replicationSinkHandler) {
1878       this.replicationSourceHandler.stopReplicationService();
1879     } else {
1880       if (this.replicationSourceHandler != null) {
1881         this.replicationSourceHandler.stopReplicationService();
1882       }
1883       if (this.replicationSinkHandler != null) {
1884         this.replicationSinkHandler.stopReplicationService();
1885       }
1886     }
1887   }
1888 
1889   /**
1890    * @return Return the object that implements the replication
1891    * source service.
1892    */
1893   ReplicationSourceService getReplicationSourceService() {
1894     return replicationSourceHandler;
1895   }
1896 
1897   /**
1898    * @return Return the object that implements the replication
1899    * sink service.
1900    */
1901   ReplicationSinkService getReplicationSinkService() {
1902     return replicationSinkHandler;
1903   }
1904 
1905   /**
1906    * Get the current master from ZooKeeper and open the RPC connection to it.
1907    *
1908    * Method will block until a master is available. You can break from this
1909    * block by requesting the server stop.
1910    *
1911    * @return master + port, or null if server has been stopped
1912    */
1913   private Pair<ServerName, RegionServerStatusService.BlockingInterface>
1914   createRegionServerStatusStub() {
1915     ServerName sn = null;
1916     long previousLogTime = 0;
1917     RegionServerStatusService.BlockingInterface master = null;
1918     boolean refresh = false; // for the first time, use cached data
1919     RegionServerStatusService.BlockingInterface intf = null;
1920     while (keepLooping() && master == null) {
1921       sn = this.masterAddressManager.getMasterAddress(refresh);
1922       if (sn == null) {
1923         if (!keepLooping()) {
1924           // give up with no connection.
1925           LOG.debug("No master found and cluster is stopped; bailing out");
1926           return null;
1927         }
1928         LOG.debug("No master found; retry");
1929         previousLogTime = System.currentTimeMillis();
1930         refresh = true; // let's try pull it from ZK directly
1931         sleeper.sleep();
1932         continue;
1933       }
1934 
1935       new InetSocketAddress(sn.getHostname(), sn.getPort());
1936       try {
1937         BlockingRpcChannel channel =
1938             this.rpcClient.createBlockingRpcChannel(sn, userProvider.getCurrent(), this.rpcTimeout);
1939         intf = RegionServerStatusService.newBlockingStub(channel);
1940         break;
1941       } catch (IOException e) {
1942         e = e instanceof RemoteException ?
1943             ((RemoteException)e).unwrapRemoteException() : e;
1944         if (e instanceof ServerNotRunningYetException) {
1945           if (System.currentTimeMillis() > (previousLogTime+1000)){
1946             LOG.info("Master isn't available yet, retrying");
1947             previousLogTime = System.currentTimeMillis();
1948           }
1949         } else {
1950           if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1951             LOG.warn("Unable to connect to master. Retrying. Error was:", e);
1952             previousLogTime = System.currentTimeMillis();
1953           }
1954         }
1955         try {
1956           Thread.sleep(200);
1957         } catch (InterruptedException ignored) {
1958         }
1959       }
1960     }
1961     return new Pair<ServerName, RegionServerStatusService.BlockingInterface>(sn, intf);
1962   }
1963 
1964   /**
1965    * @return True if we should break loop because cluster is going down or
1966    * this server has been stopped or hdfs has gone bad.
1967    */
1968   private boolean keepLooping() {
1969     return !this.stopped && isClusterUp();
1970   }
1971 
1972   /*
1973    * Let the master know we're here Run initialization using parameters passed
1974    * us by the master.
1975    * @return A Map of key/value configurations we got from the Master else
1976    * null if we failed to register.
1977    * @throws IOException
1978    */
1979   private RegionServerStartupResponse reportForDuty() throws IOException {
1980     RegionServerStartupResponse result = null;
1981     Pair<ServerName, RegionServerStatusService.BlockingInterface> p =
1982       createRegionServerStatusStub();
1983     this.rssStub = p.getSecond();
1984     ServerName masterServerName = p.getFirst();
1985     if (masterServerName == null) return result;
1986     try {
1987       this.requestCount.set(0);
1988       LOG.info("reportForDuty to master=" + masterServerName + " with port=" + this.isa.getPort() +
1989         ", startcode=" + this.startcode);
1990       long now = EnvironmentEdgeManager.currentTimeMillis();
1991       int port = this.isa.getPort();
1992       RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
1993       request.setPort(port);
1994       request.setServerStartCode(this.startcode);
1995       request.setServerCurrentTime(now);
1996       result = this.rssStub.regionServerStartup(null, request.build());
1997     } catch (ServiceException se) {
1998       IOException ioe = ProtobufUtil.getRemoteException(se);
1999       if (ioe instanceof ClockOutOfSyncException) {
2000         LOG.fatal("Master rejected startup because clock is out of sync", ioe);
2001         // Re-throw IOE will cause RS to abort
2002         throw ioe;
2003       } else if (ioe instanceof ServerNotRunningYetException) {
2004         LOG.debug("Master is not running yet");
2005       } else {
2006         LOG.warn("error telling master we are up", se);
2007       }
2008     }
2009     return result;
2010   }
2011 
2012   @Override
2013   public long getLastSequenceId(byte[] region) {
2014     Long lastFlushedSequenceId = -1l;
2015     try {
2016       GetLastFlushedSequenceIdRequest req = RequestConverter
2017           .buildGetLastFlushedSequenceIdRequest(region);
2018       lastFlushedSequenceId = rssStub.getLastFlushedSequenceId(null, req)
2019           .getLastFlushedSequenceId();
2020     } catch (ServiceException e) {
2021       lastFlushedSequenceId = -1l;
2022       LOG.warn("Unable to connect to the master to check " + "the last flushed sequence id", e);
2023     }
2024     return lastFlushedSequenceId;
2025   }
2026 
2027   /**
2028    * Closes all regions.  Called on our way out.
2029    * Assumes that its not possible for new regions to be added to onlineRegions
2030    * while this method runs.
2031    */
2032   protected void closeAllRegions(final boolean abort) {
2033     closeUserRegions(abort);
2034     closeMetaTableRegions(abort);
2035   }
2036 
2037   /**
2038    * Close meta region if we carry it
2039    * @param abort Whether we're running an abort.
2040    */
2041   void closeMetaTableRegions(final boolean abort) {
2042     HRegion meta = null;
2043     this.lock.writeLock().lock();
2044     try {
2045       for (Map.Entry<String, HRegion> e: onlineRegions.entrySet()) {
2046         HRegionInfo hri = e.getValue().getRegionInfo();
2047         if (hri.isMetaRegion()) {
2048           meta = e.getValue();
2049         }
2050         if (meta != null) break;
2051       }
2052     } finally {
2053       this.lock.writeLock().unlock();
2054     }
2055     if (meta != null) closeRegionIgnoreErrors(meta.getRegionInfo(), abort);
2056   }
2057 
2058   /**
2059    * Schedule closes on all user regions.
2060    * Should be safe calling multiple times because it wont' close regions
2061    * that are already closed or that are closing.
2062    * @param abort Whether we're running an abort.
2063    */
2064   void closeUserRegions(final boolean abort) {
2065     this.lock.writeLock().lock();
2066     try {
2067       for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
2068         HRegion r = e.getValue();
2069         if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
2070           // Don't update zk with this close transition; pass false.
2071           closeRegionIgnoreErrors(r.getRegionInfo(), abort);
2072         }
2073       }
2074     } finally {
2075       this.lock.writeLock().unlock();
2076     }
2077   }
2078 
2079   /** @return the info server */
2080   public InfoServer getInfoServer() {
2081     return infoServer;
2082   }
2083 
2084   /**
2085    * @return true if a stop has been requested.
2086    */
2087   @Override
2088   public boolean isStopped() {
2089     return this.stopped;
2090   }
2091 
2092   @Override
2093   public boolean isStopping() {
2094     return this.stopping;
2095   }
2096 
2097   @Override
2098   public Map<String, HRegion> getRecoveringRegions() {
2099     return this.recoveringRegions;
2100   }
2101 
2102   /**
2103    *
2104    * @return the configuration
2105    */
2106   @Override
2107   public Configuration getConfiguration() {
2108     return conf;
2109   }
2110 
2111   /** @return the write lock for the server */
2112   ReentrantReadWriteLock.WriteLock getWriteLock() {
2113     return lock.writeLock();
2114   }
2115 
2116   public int getNumberOfOnlineRegions() {
2117     return this.onlineRegions.size();
2118   }
2119 
2120   boolean isOnlineRegionsEmpty() {
2121     return this.onlineRegions.isEmpty();
2122   }
2123 
2124   /**
2125    * For tests, web ui and metrics.
2126    * This method will only work if HRegionServer is in the same JVM as client;
2127    * HRegion cannot be serialized to cross an rpc.
2128    */
2129   public Collection<HRegion> getOnlineRegionsLocalContext() {
2130     Collection<HRegion> regions = this.onlineRegions.values();
2131     return Collections.unmodifiableCollection(regions);
2132   }
2133 
2134   @Override
2135   public void addToOnlineRegions(HRegion region) {
2136     this.onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
2137   }
2138 
2139   /**
2140    * @return A new Map of online regions sorted by region size with the first entry being the
2141    * biggest.  If two regions are the same size, then the last one found wins; i.e. this method
2142    * may NOT return all regions.
2143    */
2144   SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedBySize() {
2145     // we'll sort the regions in reverse
2146     SortedMap<Long, HRegion> sortedRegions = new TreeMap<Long, HRegion>(
2147         new Comparator<Long>() {
2148           @Override
2149           public int compare(Long a, Long b) {
2150             return -1 * a.compareTo(b);
2151           }
2152         });
2153     // Copy over all regions. Regions are sorted by size with biggest first.
2154     for (HRegion region : this.onlineRegions.values()) {
2155       sortedRegions.put(region.memstoreSize.get(), region);
2156     }
2157     return sortedRegions;
2158   }
2159 
2160   /**
2161    * @return time stamp in millis of when this region server was started
2162    */
2163   public long getStartcode() {
2164     return this.startcode;
2165   }
2166 
2167   /** @return reference to FlushRequester */
2168   @Override
2169   public FlushRequester getFlushRequester() {
2170     return this.cacheFlusher;
2171   }
2172 
2173   /**
2174    * Get the top N most loaded regions this server is serving so we can tell the
2175    * master which regions it can reallocate if we're overloaded. TODO: actually
2176    * calculate which regions are most loaded. (Right now, we're just grabbing
2177    * the first N regions being served regardless of load.)
2178    */
2179   protected HRegionInfo[] getMostLoadedRegions() {
2180     ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2181     for (HRegion r : onlineRegions.values()) {
2182       if (!r.isAvailable()) {
2183         continue;
2184       }
2185       if (regions.size() < numRegionsToReport) {
2186         regions.add(r.getRegionInfo());
2187       } else {
2188         break;
2189       }
2190     }
2191     return regions.toArray(new HRegionInfo[regions.size()]);
2192   }
2193 
2194   @Override
2195   public Leases getLeases() {
2196     return leases;
2197   }
2198 
2199   /**
2200    * @return Return the rootDir.
2201    */
2202   protected Path getRootDir() {
2203     return rootDir;
2204   }
2205 
2206   /**
2207    * @return Return the fs.
2208    */
2209   @Override
2210   public FileSystem getFileSystem() {
2211     return fs;
2212   }
2213 
2214   @Override
2215   public String toString() {
2216     return getServerName().toString();
2217   }
2218 
2219   /**
2220    * Interval at which threads should run
2221    *
2222    * @return the interval
2223    */
2224   public int getThreadWakeFrequency() {
2225     return threadWakeFrequency;
2226   }
2227 
2228   @Override
2229   public ZooKeeperWatcher getZooKeeper() {
2230     return zooKeeper;
2231   }
2232 
2233   @Override
2234   public ServerName getServerName() {
2235     // Our servername could change after we talk to the master.
2236     return this.serverNameFromMasterPOV == null?
2237         ServerName.valueOf(this.isa.getHostName(), this.isa.getPort(), this.startcode) :
2238         this.serverNameFromMasterPOV;
2239   }
2240 
2241   @Override
2242   public CompactionRequestor getCompactionRequester() {
2243     return this.compactSplitThread;
2244   }
2245 
2246   public ZooKeeperWatcher getZooKeeperWatcher() {
2247     return this.zooKeeper;
2248   }
2249 
2250   public RegionServerCoprocessorHost getCoprocessorHost(){
2251     return this.rsHost;
2252   }
2253 
2254   @Override
2255   public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
2256     return this.regionsInTransitionInRS;
2257   }
2258 
2259   @Override
2260   public ExecutorService getExecutorService() {
2261     return service;
2262   }
2263 
2264   //
2265   // Main program and support routines
2266   //
2267 
2268   /**
2269    * Load the replication service objects, if any
2270    */
2271   static private void createNewReplicationInstance(Configuration conf,
2272     HRegionServer server, FileSystem fs, Path logDir, Path oldLogDir) throws IOException{
2273 
2274     // If replication is not enabled, then return immediately.
2275     if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
2276         HConstants.REPLICATION_ENABLE_DEFAULT)) {
2277       return;
2278     }
2279 
2280     // read in the name of the source replication class from the config file.
2281     String sourceClassname = conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,
2282                                HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2283 
2284     // read in the name of the sink replication class from the config file.
2285     String sinkClassname = conf.get(HConstants.REPLICATION_SINK_SERVICE_CLASSNAME,
2286                              HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2287 
2288     // If both the sink and the source class names are the same, then instantiate
2289     // only one object.
2290     if (sourceClassname.equals(sinkClassname)) {
2291       server.replicationSourceHandler = (ReplicationSourceService)
2292                                          newReplicationInstance(sourceClassname,
2293                                          conf, server, fs, logDir, oldLogDir);
2294       server.replicationSinkHandler = (ReplicationSinkService)
2295                                          server.replicationSourceHandler;
2296     } else {
2297       server.replicationSourceHandler = (ReplicationSourceService)
2298                                          newReplicationInstance(sourceClassname,
2299                                          conf, server, fs, logDir, oldLogDir);
2300       server.replicationSinkHandler = (ReplicationSinkService)
2301                                          newReplicationInstance(sinkClassname,
2302                                          conf, server, fs, logDir, oldLogDir);
2303     }
2304   }
2305 
2306   static private ReplicationService newReplicationInstance(String classname,
2307     Configuration conf, HRegionServer server, FileSystem fs, Path logDir,
2308     Path oldLogDir) throws IOException{
2309 
2310     Class<?> clazz = null;
2311     try {
2312       ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
2313       clazz = Class.forName(classname, true, classLoader);
2314     } catch (java.lang.ClassNotFoundException nfe) {
2315       throw new IOException("Could not find class for " + classname);
2316     }
2317 
2318     // create an instance of the replication object.
2319     ReplicationService service = (ReplicationService)
2320                               ReflectionUtils.newInstance(clazz, conf);
2321     service.initialize(server, fs, logDir, oldLogDir);
2322     return service;
2323   }
2324 
2325   /**
2326    * @param hrs
2327    * @return Thread the RegionServer is running in correctly named.
2328    * @throws IOException
2329    */
2330   public static Thread startRegionServer(final HRegionServer hrs)
2331       throws IOException {
2332     return startRegionServer(hrs, "regionserver" + hrs.isa.getPort());
2333   }
2334 
2335   /**
2336    * @param hrs
2337    * @param name
2338    * @return Thread the RegionServer is running in correctly named.
2339    * @throws IOException
2340    */
2341   public static Thread startRegionServer(final HRegionServer hrs,
2342       final String name) throws IOException {
2343     Thread t = new Thread(hrs);
2344     t.setName(name);
2345     t.start();
2346     // Install shutdown hook that will catch signals and run an orderly shutdown
2347     // of the hrs.
2348     ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs
2349         .getConfiguration()), hrs, t);
2350     return t;
2351   }
2352 
2353   /**
2354    * Utility for constructing an instance of the passed HRegionServer class.
2355    *
2356    * @param regionServerClass
2357    * @param conf2
2358    * @return HRegionServer instance.
2359    */
2360   public static HRegionServer constructRegionServer(
2361       Class<? extends HRegionServer> regionServerClass,
2362       final Configuration conf2) {
2363     try {
2364       Constructor<? extends HRegionServer> c = regionServerClass
2365           .getConstructor(Configuration.class);
2366       return c.newInstance(conf2);
2367     } catch (Exception e) {
2368       throw new RuntimeException("Failed construction of " + "Regionserver: "
2369           + regionServerClass.toString(), e);
2370     }
2371   }
2372 
2373   /**
2374    * @see org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine
2375    */
2376   public static void main(String[] args) throws Exception {
2377 	VersionInfo.logVersion();
2378     Configuration conf = HBaseConfiguration.create();
2379     @SuppressWarnings("unchecked")
2380     Class<? extends HRegionServer> regionServerClass = (Class<? extends HRegionServer>) conf
2381         .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
2382 
2383     new HRegionServerCommandLine(regionServerClass).doMain(args);
2384   }
2385 
2386   /**
2387    * Gets the online regions of the specified table.
2388    * This method looks at the in-memory onlineRegions.  It does not go to <code>hbase:meta</code>.
2389    * Only returns <em>online</em> regions.  If a region on this table has been
2390    * closed during a disable, etc., it will not be included in the returned list.
2391    * So, the returned list may not necessarily be ALL regions in this table, its
2392    * all the ONLINE regions in the table.
2393    * @param tableName
2394    * @return Online regions from <code>tableName</code>
2395    */
2396   @Override
2397   public List<HRegion> getOnlineRegions(TableName tableName) {
2398      List<HRegion> tableRegions = new ArrayList<HRegion>();
2399      synchronized (this.onlineRegions) {
2400        for (HRegion region: this.onlineRegions.values()) {
2401          HRegionInfo regionInfo = region.getRegionInfo();
2402          if(regionInfo.getTable().equals(tableName)) {
2403            tableRegions.add(region);
2404          }
2405        }
2406      }
2407      return tableRegions;
2408    }
2409 
2410   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
2411   public String[] getCoprocessors() {
2412     TreeSet<String> coprocessors = new TreeSet<String>(
2413         this.hlog.getCoprocessorHost().getCoprocessors());
2414     Collection<HRegion> regions = getOnlineRegionsLocalContext();
2415     for (HRegion region: regions) {
2416       coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
2417     }
2418     return coprocessors.toArray(new String[coprocessors.size()]);
2419   }
2420 
2421   /**
2422    * Instantiated as a scanner lease. If the lease times out, the scanner is
2423    * closed
2424    */
2425   private class ScannerListener implements LeaseListener {
2426     private final String scannerName;
2427 
2428     ScannerListener(final String n) {
2429       this.scannerName = n;
2430     }
2431 
2432     @Override
2433     public void leaseExpired() {
2434       RegionScannerHolder rsh = scanners.remove(this.scannerName);
2435       if (rsh != null) {
2436         RegionScanner s = rsh.s;
2437         LOG.info("Scanner " + this.scannerName + " lease expired on region "
2438             + s.getRegionInfo().getRegionNameAsString());
2439         try {
2440           HRegion region = getRegion(s.getRegionInfo().getRegionName());
2441           if (region != null && region.getCoprocessorHost() != null) {
2442             region.getCoprocessorHost().preScannerClose(s);
2443           }
2444 
2445           s.close();
2446           if (region != null && region.getCoprocessorHost() != null) {
2447             region.getCoprocessorHost().postScannerClose(s);
2448           }
2449         } catch (IOException e) {
2450           LOG.error("Closing scanner for "
2451               + s.getRegionInfo().getRegionNameAsString(), e);
2452         }
2453       } else {
2454         LOG.info("Scanner " + this.scannerName + " lease expired");
2455       }
2456     }
2457   }
2458 
2459   /**
2460    * Called to verify that this server is up and running.
2461    *
2462    * @throws IOException
2463    */
2464   protected void checkOpen() throws IOException {
2465     if (this.stopped || this.abortRequested) {
2466       throw new RegionServerStoppedException("Server " + getServerName() +
2467         " not running" + (this.abortRequested ? ", aborting" : ""));
2468     }
2469     if (!fsOk) {
2470       throw new RegionServerStoppedException("File system not available");
2471     }
2472   }
2473 
2474 
2475   /**
2476    * Try to close the region, logs a warning on failure but continues.
2477    * @param region Region to close
2478    */
2479   private void closeRegionIgnoreErrors(HRegionInfo region, final boolean abort) {
2480     try {
2481       if (!closeRegion(region.getEncodedName(), abort, false, -1, null)) {
2482         LOG.warn("Failed to close " + region.getRegionNameAsString() +
2483             " - ignoring and continuing");
2484       }
2485     } catch (IOException e) {
2486       LOG.warn("Failed to close " + region.getRegionNameAsString() +
2487           " - ignoring and continuing", e);
2488     }
2489   }
2490 
2491   /**
2492    * Close asynchronously a region, can be called from the master or internally by the regionserver
2493    * when stopping. If called from the master, the region will update the znode status.
2494    *
2495    * <p>
2496    * If an opening was in progress, this method will cancel it, but will not start a new close. The
2497    * coprocessors are not called in this case. A NotServingRegionException exception is thrown.
2498    * </p>
2499 
2500    * <p>
2501    *   If a close was in progress, this new request will be ignored, and an exception thrown.
2502    * </p>
2503    *
2504    * @param encodedName Region to close
2505    * @param abort True if we are aborting
2506    * @param zk True if we are to update zk about the region close; if the close
2507    * was orchestrated by master, then update zk.  If the close is being run by
2508    * the regionserver because its going down, don't update zk.
2509    * @param versionOfClosingNode the version of znode to compare when RS transitions the znode from
2510    *   CLOSING state.
2511    * @return True if closed a region.
2512    * @throws NotServingRegionException if the region is not online
2513    * @throws RegionAlreadyInTransitionException if the region is already closing
2514    */
2515   protected boolean closeRegion(String encodedName, final boolean abort,
2516       final boolean zk, final int versionOfClosingNode, final ServerName sn)
2517       throws NotServingRegionException, RegionAlreadyInTransitionException {
2518     //Check for permissions to close.
2519     HRegion actualRegion = this.getFromOnlineRegions(encodedName);
2520     if ((actualRegion != null) && (actualRegion.getCoprocessorHost() != null)) {
2521       try {
2522         actualRegion.getCoprocessorHost().preClose(false);
2523       } catch (IOException exp) {
2524         LOG.warn("Unable to close region: the coprocessor launched an error ", exp);
2525         return false;
2526       }
2527     }
2528 
2529     final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(encodedName.getBytes(),
2530         Boolean.FALSE);
2531 
2532     if (Boolean.TRUE.equals(previous)) {
2533       LOG.info("Received CLOSE for the region:" + encodedName + " , which we are already " +
2534           "trying to OPEN. Cancelling OPENING.");
2535       if (!regionsInTransitionInRS.replace(encodedName.getBytes(), previous, Boolean.FALSE)){
2536         // The replace failed. That should be an exceptional case, but theoretically it can happen.
2537         // We're going to try to do a standard close then.
2538         LOG.warn("The opening for region " + encodedName + " was done before we could cancel it." +
2539             " Doing a standard close now");
2540         return closeRegion(encodedName, abort, zk, versionOfClosingNode, sn);
2541       }
2542       // Let's get the region from the online region list again
2543       actualRegion = this.getFromOnlineRegions(encodedName);
2544       if (actualRegion == null) { // If already online, we still need to close it.
2545         LOG.info("The opening previously in progress has been cancelled by a CLOSE request.");
2546         // The master deletes the znode when it receives this exception.
2547         throw new NotServingRegionException("The region " + encodedName +
2548           " was opening but not yet served. Opening is cancelled.");
2549       }
2550     } else if (Boolean.FALSE.equals(previous)) {
2551       LOG.info("Received CLOSE for the region: " + encodedName +
2552         " ,which we are already trying to CLOSE, but not completed yet");
2553       // The master will retry till the region is closed. We need to do this since
2554       // the region could fail to close somehow. If we mark the region closed in master
2555       // while it is not, there could be data loss.
2556       // If the region stuck in closing for a while, and master runs out of retries,
2557       // master will move the region to failed_to_close. Later on, if the region
2558       // is indeed closed, master can properly re-assign it.
2559       throw new RegionAlreadyInTransitionException("The region " + encodedName +
2560         " was already closing. New CLOSE request is ignored.");
2561     }
2562 
2563     if (actualRegion == null) {
2564       LOG.error("Received CLOSE for a region which is not online, and we're not opening.");
2565       this.regionsInTransitionInRS.remove(encodedName.getBytes());
2566       // The master deletes the znode when it receives this exception.
2567       throw new NotServingRegionException("The region " + encodedName +
2568           " is not online, and is not opening.");
2569     }
2570 
2571     CloseRegionHandler crh;
2572     final HRegionInfo hri = actualRegion.getRegionInfo();
2573     if (hri.isMetaRegion()) {
2574       crh = new CloseMetaHandler(this, this, hri, abort, zk, versionOfClosingNode);
2575     } else {
2576       crh = new CloseRegionHandler(this, this, hri, abort, zk, versionOfClosingNode, sn);
2577     }
2578     this.service.submit(crh);
2579     return true;
2580   }
2581 
2582    /**
2583    * @param regionName
2584    * @return HRegion for the passed binary <code>regionName</code> or null if
2585    *         named region is not member of the online regions.
2586    */
2587   public HRegion getOnlineRegion(final byte[] regionName) {
2588     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
2589     return this.onlineRegions.get(encodedRegionName);
2590   }
2591 
2592   public InetSocketAddress[] getRegionBlockLocations(final String encodedRegionName) {
2593     return this.regionFavoredNodesMap.get(encodedRegionName);
2594   }
2595 
2596   @Override
2597   public HRegion getFromOnlineRegions(final String encodedRegionName) {
2598     return this.onlineRegions.get(encodedRegionName);
2599   }
2600 
2601 
2602   @Override
2603   public boolean removeFromOnlineRegions(final HRegion r, ServerName destination) {
2604     HRegion toReturn = this.onlineRegions.remove(r.getRegionInfo().getEncodedName());
2605 
2606     if (destination != null) {
2607       HLog wal = getWAL();
2608       long closeSeqNum = wal.getEarliestMemstoreSeqNum(r.getRegionInfo().getEncodedNameAsBytes());
2609       if (closeSeqNum == HConstants.NO_SEQNUM) {
2610         // No edits in WAL for this region; get the sequence number when the region was opened.
2611         closeSeqNum = r.getOpenSeqNum();
2612         if (closeSeqNum == HConstants.NO_SEQNUM) {
2613           closeSeqNum = 0;
2614         }
2615       }
2616       addToMovedRegions(r.getRegionInfo().getEncodedName(), destination, closeSeqNum);
2617     }
2618     this.regionFavoredNodesMap.remove(r.getRegionInfo().getEncodedName());
2619     return toReturn != null;
2620   }
2621 
2622   /**
2623    * Protected utility method for safely obtaining an HRegion handle.
2624    *
2625    * @param regionName
2626    *          Name of online {@link HRegion} to return
2627    * @return {@link HRegion} for <code>regionName</code>
2628    * @throws NotServingRegionException
2629    */
2630   protected HRegion getRegion(final byte[] regionName)
2631       throws NotServingRegionException {
2632     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
2633     return getRegionByEncodedName(regionName, encodedRegionName);
2634   }
2635 
2636   protected HRegion getRegionByEncodedName(String encodedRegionName)
2637       throws NotServingRegionException {
2638     return getRegionByEncodedName(null, encodedRegionName);
2639   }
2640 
2641   protected HRegion getRegionByEncodedName(byte[] regionName, String encodedRegionName)
2642     throws NotServingRegionException {
2643     HRegion region = this.onlineRegions.get(encodedRegionName);
2644     if (region == null) {
2645       MovedRegionInfo moveInfo = getMovedRegion(encodedRegionName);
2646       if (moveInfo != null) {
2647         throw new RegionMovedException(moveInfo.getServerName(), moveInfo.getSeqNum());
2648       }
2649       Boolean isOpening = this.regionsInTransitionInRS.get(Bytes.toBytes(encodedRegionName));
2650       String regionNameStr = regionName == null?
2651         encodedRegionName: Bytes.toStringBinary(regionName);
2652       if (isOpening != null && isOpening.booleanValue()) {
2653         throw new RegionOpeningException("Region " + regionNameStr + " is opening");
2654       }
2655       throw new NotServingRegionException("Region " + regionNameStr + " is not online");
2656     }
2657     return region;
2658   }
2659 
2660   /*
2661    * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
2662    * IOE if it isn't already.
2663    *
2664    * @param t Throwable
2665    *
2666    * @return Throwable converted to an IOE; methods can only let out IOEs.
2667    */
2668   protected Throwable cleanup(final Throwable t) {
2669     return cleanup(t, null);
2670   }
2671 
2672   /*
2673    * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
2674    * IOE if it isn't already.
2675    *
2676    * @param t Throwable
2677    *
2678    * @param msg Message to log in error. Can be null.
2679    *
2680    * @return Throwable converted to an IOE; methods can only let out IOEs.
2681    */
2682   protected Throwable cleanup(final Throwable t, final String msg) {
2683     // Don't log as error if NSRE; NSRE is 'normal' operation.
2684     if (t instanceof NotServingRegionException) {
2685       LOG.debug("NotServingRegionException; " + t.getMessage());
2686       return t;
2687     }
2688     if (msg == null) {
2689       LOG.error("", RemoteExceptionHandler.checkThrowable(t));
2690     } else {
2691       LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));
2692     }
2693     if (!checkOOME(t)) {
2694       checkFileSystem();
2695     }
2696     return t;
2697   }
2698 
2699   /*
2700    * @param t
2701    *
2702    * @param msg Message to put in new IOE if passed <code>t</code> is not an IOE
2703    *
2704    * @return Make <code>t</code> an IOE if it isn't already.
2705    */
2706   protected IOException convertThrowableToIOE(final Throwable t, final String msg) {
2707     return (t instanceof IOException ? (IOException) t : msg == null
2708         || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
2709   }
2710 
2711   /*
2712    * Check if an OOME and, if so, abort immediately to avoid creating more objects.
2713    *
2714    * @param e
2715    *
2716    * @return True if we OOME'd and are aborting.
2717    */
2718   @Override
2719   public boolean checkOOME(final Throwable e) {
2720     boolean stop = false;
2721     try {
2722       if (e instanceof OutOfMemoryError
2723           || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
2724           || (e.getMessage() != null && e.getMessage().contains(
2725               "java.lang.OutOfMemoryError"))) {
2726         stop = true;
2727         LOG.fatal(
2728           "Run out of memory; HRegionServer will abort itself immediately", e);
2729       }
2730     } finally {
2731       if (stop) {
2732         Runtime.getRuntime().halt(1);
2733       }
2734     }
2735     return stop;
2736   }
2737 
2738   /**
2739    * Checks to see if the file system is still accessible. If not, sets
2740    * abortRequested and stopRequested
2741    *
2742    * @return false if file system is not available
2743    */
2744   public boolean checkFileSystem() {
2745     if (this.fsOk && this.fs != null) {
2746       try {
2747         FSUtils.checkFileSystemAvailable(this.fs);
2748       } catch (IOException e) {
2749         abort("File System not available", e);
2750         this.fsOk = false;
2751       }
2752     }
2753     return this.fsOk;
2754   }
2755 
2756   protected long addScanner(RegionScanner s, HRegion r) throws LeaseStillHeldException {
2757     long scannerId = -1;
2758     while (true) {
2759       scannerId = Math.abs(rand.nextLong() << 24) ^ startcode;
2760       String scannerName = String.valueOf(scannerId);
2761       RegionScannerHolder existing =
2762         scanners.putIfAbsent(scannerName, new RegionScannerHolder(s, r));
2763       if (existing == null) {
2764         this.leases.createLease(scannerName, this.scannerLeaseTimeoutPeriod,
2765           new ScannerListener(scannerName));
2766         break;
2767       }
2768     }
2769     return scannerId;
2770   }
2771 
2772   /**
2773    * Generate a random positive long number
2774    *
2775    * @return a random positive long number
2776    */
2777   protected long nextLong() {
2778     long n = rand.nextLong();
2779     if (n == 0) {
2780       return nextLong();
2781     }
2782     if (n < 0) {
2783       n = -n;
2784     }
2785     return n;
2786   }
2787 
2788   // Start Client methods
2789 
2790   /**
2791    * Get data from a table.
2792    *
2793    * @param controller the RPC controller
2794    * @param request the get request
2795    * @throws ServiceException
2796    */
2797   @Override
2798   public GetResponse get(final RpcController controller,
2799       final GetRequest request) throws ServiceException {
2800     long before = EnvironmentEdgeManager.currentTimeMillis();
2801     try {
2802       checkOpen();
2803       requestCount.increment();
2804       HRegion region = getRegion(request.getRegion());
2805 
2806       GetResponse.Builder builder = GetResponse.newBuilder();
2807       ClientProtos.Get get = request.getGet();
2808       Boolean existence = null;
2809       Result r = null;
2810 
2811       if (get.hasClosestRowBefore() && get.getClosestRowBefore()) {
2812         if (get.getColumnCount() != 1) {
2813           throw new DoNotRetryIOException(
2814             "get ClosestRowBefore supports one and only one family now, not "
2815               + get.getColumnCount() + " families");
2816         }
2817         byte[] row = get.getRow().toByteArray();
2818         byte[] family = get.getColumn(0).getFamily().toByteArray();
2819         r = region.getClosestRowBefore(row, family);
2820       } else {
2821         Get clientGet = ProtobufUtil.toGet(get);
2822         if (get.getExistenceOnly() && region.getCoprocessorHost() != null) {
2823           existence = region.getCoprocessorHost().preExists(clientGet);
2824         }
2825         if (existence == null) {
2826           r = region.get(clientGet);
2827           if (get.getExistenceOnly()) {
2828             boolean exists = r.getExists();
2829             if (region.getCoprocessorHost() != null) {
2830               exists = region.getCoprocessorHost().postExists(clientGet, exists);
2831             }
2832             existence = exists;
2833           }
2834         }
2835       }
2836       if (existence != null){
2837         ClientProtos.Result pbr = ProtobufUtil.toResult(existence);
2838         builder.setResult(pbr);
2839       } else  if (r != null) {
2840         ClientProtos.Result pbr = ProtobufUtil.toResult(r);
2841         builder.setResult(pbr);
2842       }
2843       return builder.build();
2844     } catch (IOException ie) {
2845       throw new ServiceException(ie);
2846     } finally {
2847       metricsRegionServer.updateGet(EnvironmentEdgeManager.currentTimeMillis() - before);
2848     }
2849   }
2850 
2851 
2852   /**
2853    * Mutate data in a table.
2854    *
2855    * @param rpcc the RPC controller
2856    * @param request the mutate request
2857    * @throws ServiceException
2858    */
2859   @Override
2860   public MutateResponse mutate(final RpcController rpcc,
2861       final MutateRequest request) throws ServiceException {
2862     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
2863     // It is also the conduit via which we pass back data.
2864     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
2865     CellScanner cellScanner = controller != null? controller.cellScanner(): null;
2866     // Clear scanner so we are not holding on to reference across call.
2867     if (controller != null) controller.setCellScanner(null);
2868     try {
2869       checkOpen();
2870       requestCount.increment();
2871       HRegion region = getRegion(request.getRegion());
2872       MutateResponse.Builder builder = MutateResponse.newBuilder();
2873       MutationProto mutation = request.getMutation();
2874       if (!region.getRegionInfo().isMetaTable()) {
2875         cacheFlusher.reclaimMemStoreMemory();
2876       }
2877       long nonceGroup = request.hasNonceGroup()
2878           ? request.getNonceGroup() : HConstants.NO_NONCE;
2879       Result r = null;
2880       Boolean processed = null;
2881       MutationType type = mutation.getMutateType();
2882       switch (type) {
2883       case APPEND:
2884         // TODO: this doesn't actually check anything.
2885         r = append(region, mutation, cellScanner, nonceGroup);
2886         break;
2887       case INCREMENT:
2888         // TODO: this doesn't actually check anything.
2889         r = increment(region, mutation, cellScanner, nonceGroup);
2890         break;
2891       case PUT:
2892         Put put = ProtobufUtil.toPut(mutation, cellScanner);
2893         if (request.hasCondition()) {
2894           Condition condition = request.getCondition();
2895           byte[] row = condition.getRow().toByteArray();
2896           byte[] family = condition.getFamily().toByteArray();
2897           byte[] qualifier = condition.getQualifier().toByteArray();
2898           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2899           ByteArrayComparable comparator =
2900             ProtobufUtil.toComparator(condition.getComparator());
2901           if (region.getCoprocessorHost() != null) {
2902             processed = region.getCoprocessorHost().preCheckAndPut(
2903               row, family, qualifier, compareOp, comparator, put);
2904           }
2905           if (processed == null) {
2906             boolean result = region.checkAndMutate(row, family,
2907               qualifier, compareOp, comparator, put, true);
2908             if (region.getCoprocessorHost() != null) {
2909               result = region.getCoprocessorHost().postCheckAndPut(row, family,
2910                 qualifier, compareOp, comparator, put, result);
2911             }
2912             processed = result;
2913           }
2914         } else {
2915           region.put(put);
2916           processed = Boolean.TRUE;
2917         }
2918         break;
2919       case DELETE:
2920         Delete delete = ProtobufUtil.toDelete(mutation, cellScanner);
2921         if (request.hasCondition()) {
2922           Condition condition = request.getCondition();
2923           byte[] row = condition.getRow().toByteArray();
2924           byte[] family = condition.getFamily().toByteArray();
2925           byte[] qualifier = condition.getQualifier().toByteArray();
2926           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2927           ByteArrayComparable comparator =
2928             ProtobufUtil.toComparator(condition.getComparator());
2929           if (region.getCoprocessorHost() != null) {
2930             processed = region.getCoprocessorHost().preCheckAndDelete(
2931               row, family, qualifier, compareOp, comparator, delete);
2932           }
2933           if (processed == null) {
2934             boolean result = region.checkAndMutate(row, family,
2935               qualifier, compareOp, comparator, delete, true);
2936             if (region.getCoprocessorHost() != null) {
2937               result = region.getCoprocessorHost().postCheckAndDelete(row, family,
2938                 qualifier, compareOp, comparator, delete, result);
2939             }
2940             processed = result;
2941           }
2942         } else {
2943           region.delete(delete);
2944           processed = Boolean.TRUE;
2945         }
2946         break;
2947         default:
2948           throw new DoNotRetryIOException(
2949             "Unsupported mutate type: " + type.name());
2950       }
2951       if (processed != null) builder.setProcessed(processed.booleanValue());
2952       addResult(builder, r, controller);
2953       return builder.build();
2954     } catch (IOException ie) {
2955       checkFileSystem();
2956       throw new ServiceException(ie);
2957     }
2958   }
2959 
2960 
2961   /**
2962    * @return True if current call supports cellblocks
2963    */
2964   private boolean isClientCellBlockSupport() {
2965     RpcCallContext context = RpcServer.getCurrentCall();
2966     return context != null && context.isClientCellBlockSupport();
2967   }
2968 
2969   private void addResult(final MutateResponse.Builder builder,
2970       final Result result, final PayloadCarryingRpcController rpcc) {
2971     if (result == null) return;
2972     if (isClientCellBlockSupport()) {
2973       builder.setResult(ProtobufUtil.toResultNoData(result));
2974       rpcc.setCellScanner(result.cellScanner());
2975     } else {
2976       ClientProtos.Result pbr = ProtobufUtil.toResult(result);
2977       builder.setResult(pbr);
2978     }
2979   }
2980 
2981   //
2982   // remote scanner interface
2983   //
2984 
2985   /**
2986    * Scan data in a table.
2987    *
2988    * @param controller the RPC controller
2989    * @param request the scan request
2990    * @throws ServiceException
2991    */
2992   @Override
2993   public ScanResponse scan(final RpcController controller, final ScanRequest request)
2994   throws ServiceException {
2995     Leases.Lease lease = null;
2996     String scannerName = null;
2997     try {
2998       if (!request.hasScannerId() && !request.hasScan()) {
2999         throw new DoNotRetryIOException(
3000           "Missing required input: scannerId or scan");
3001       }
3002       long scannerId = -1;
3003       if (request.hasScannerId()) {
3004         scannerId = request.getScannerId();
3005         scannerName = String.valueOf(scannerId);
3006       }
3007       try {
3008         checkOpen();
3009       } catch (IOException e) {
3010         // If checkOpen failed, server not running or filesystem gone,
3011         // cancel this lease; filesystem is gone or we're closing or something.
3012         if (scannerName != null) {
3013           try {
3014             leases.cancelLease(scannerName);
3015           } catch (LeaseException le) {
3016             LOG.info("Server shutting down and client tried to access missing scanner " +
3017               scannerName);
3018           }
3019         }
3020         throw e;
3021       }
3022       requestCount.increment();
3023 
3024       int ttl = 0;
3025       HRegion region = null;
3026       RegionScanner scanner = null;
3027       RegionScannerHolder rsh = null;
3028       boolean moreResults = true;
3029       boolean closeScanner = false;
3030       ScanResponse.Builder builder = ScanResponse.newBuilder();
3031       if (request.hasCloseScanner()) {
3032         closeScanner = request.getCloseScanner();
3033       }
3034       int rows = 1;
3035       if (request.hasNumberOfRows()) {
3036         rows = request.getNumberOfRows();
3037       }
3038       if (request.hasScannerId()) {
3039         rsh = scanners.get(scannerName);
3040         if (rsh == null) {
3041           LOG.info("Client tried to access missing scanner " + scannerName);
3042           throw new UnknownScannerException(
3043             "Name: " + scannerName + ", already closed?");
3044         }
3045         scanner = rsh.s;
3046         HRegionInfo hri = scanner.getRegionInfo();
3047         region = getRegion(hri.getRegionName());
3048         if (region != rsh.r) { // Yes, should be the same instance
3049           throw new NotServingRegionException("Region was re-opened after the scanner"
3050             + scannerName + " was created: " + hri.getRegionNameAsString());
3051         }
3052       } else {
3053         region = getRegion(request.getRegion());
3054         ClientProtos.Scan protoScan = request.getScan();
3055         boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
3056         Scan scan = ProtobufUtil.toScan(protoScan);
3057         // if the request doesn't set this, get the default region setting.
3058         if (!isLoadingCfsOnDemandSet) {
3059           scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
3060         }
3061         scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
3062         region.prepareScanner(scan);
3063         if (region.getCoprocessorHost() != null) {
3064           scanner = region.getCoprocessorHost().preScannerOpen(scan);
3065         }
3066         if (scanner == null) {
3067           scanner = region.getScanner(scan);
3068         }
3069         if (region.getCoprocessorHost() != null) {
3070           scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
3071         }
3072         scannerId = addScanner(scanner, region);
3073         scannerName = String.valueOf(scannerId);
3074         ttl = this.scannerLeaseTimeoutPeriod;
3075       }
3076 
3077       if (rows > 0) {
3078         // if nextCallSeq does not match throw Exception straight away. This needs to be
3079         // performed even before checking of Lease.
3080         // See HBASE-5974
3081         if (request.hasNextCallSeq()) {
3082           if (rsh == null) {
3083             rsh = scanners.get(scannerName);
3084           }
3085           if (rsh != null) {
3086             if (request.getNextCallSeq() != rsh.nextCallSeq) {
3087               throw new OutOfOrderScannerNextException("Expected nextCallSeq: " + rsh.nextCallSeq
3088                 + " But the nextCallSeq got from client: " + request.getNextCallSeq() +
3089                 "; request=" + TextFormat.shortDebugString(request));
3090             }
3091             // Increment the nextCallSeq value which is the next expected from client.
3092             rsh.nextCallSeq++;
3093           }
3094         }
3095         try {
3096           // Remove lease while its being processed in server; protects against case
3097           // where processing of request takes > lease expiration time.
3098           lease = leases.removeLease(scannerName);
3099           List<Result> results = new ArrayList<Result>(rows);
3100           long currentScanResultSize = 0;
3101 
3102           boolean done = false;
3103           // Call coprocessor. Get region info from scanner.
3104           if (region != null && region.getCoprocessorHost() != null) {
3105             Boolean bypass = region.getCoprocessorHost().preScannerNext(
3106               scanner, results, rows);
3107             if (!results.isEmpty()) {
3108               for (Result r : results) {
3109                 if (maxScannerResultSize < Long.MAX_VALUE){
3110                   for (Cell kv : r.rawCells()) {
3111                     // TODO
3112                     currentScanResultSize += KeyValueUtil.ensureKeyValue(kv).heapSize();
3113                   }
3114                 }
3115               }
3116             }
3117             if (bypass != null && bypass.booleanValue()) {
3118               done = true;
3119             }
3120           }
3121 
3122           if (!done) {
3123             long maxResultSize = scanner.getMaxResultSize();
3124             if (maxResultSize <= 0) {
3125               maxResultSize = maxScannerResultSize;
3126             }
3127             List<Cell> values = new ArrayList<Cell>();
3128             region.startRegionOperation(Operation.SCAN);
3129             try {
3130               int i = 0;
3131               synchronized(scanner) {
3132                 for (; i < rows
3133                     && currentScanResultSize < maxResultSize; ) {
3134                   // Collect values to be returned here
3135                   boolean moreRows = scanner.nextRaw(values);
3136                   if (!values.isEmpty()) {
3137                     if (maxScannerResultSize < Long.MAX_VALUE){
3138                       for (Cell kv : values) {
3139                         currentScanResultSize += KeyValueUtil.ensureKeyValue(kv).heapSize();
3140                       }
3141                     }
3142                     results.add(Result.create(values));
3143                     i++;
3144                   }
3145                   if (!moreRows) {
3146                     break;
3147                   }
3148                   values.clear();
3149                 }
3150               }
3151               region.readRequestsCount.add(i);
3152             } finally {
3153               region.closeRegionOperation();
3154             }
3155 
3156             // coprocessor postNext hook
3157             if (region != null && region.getCoprocessorHost() != null) {
3158               region.getCoprocessorHost().postScannerNext(scanner, results, rows, true);
3159             }
3160           }
3161 
3162           // If the scanner's filter - if any - is done with the scan
3163           // and wants to tell the client to stop the scan. This is done by passing
3164           // a null result, and setting moreResults to false.
3165           if (scanner.isFilterDone() && results.isEmpty()) {
3166             moreResults = false;
3167             results = null;
3168           } else {
3169             addResults(builder, results, controller);
3170           }
3171         } finally {
3172           // We're done. On way out re-add the above removed lease.
3173           // Adding resets expiration time on lease.
3174           if (scanners.containsKey(scannerName)) {
3175             if (lease != null) leases.addLease(lease);
3176             ttl = this.scannerLeaseTimeoutPeriod;
3177           }
3178         }
3179       }
3180 
3181       if (!moreResults || closeScanner) {
3182         ttl = 0;
3183         moreResults = false;
3184         if (region != null && region.getCoprocessorHost() != null) {
3185           if (region.getCoprocessorHost().preScannerClose(scanner)) {
3186             return builder.build(); // bypass
3187           }
3188         }
3189         rsh = scanners.remove(scannerName);
3190         if (rsh != null) {
3191           scanner = rsh.s;
3192           scanner.close();
3193           leases.cancelLease(scannerName);
3194           if (region != null && region.getCoprocessorHost() != null) {
3195             region.getCoprocessorHost().postScannerClose(scanner);
3196           }
3197         }
3198       }
3199 
3200       if (ttl > 0) {
3201         builder.setTtl(ttl);
3202       }
3203       builder.setScannerId(scannerId);
3204       builder.setMoreResults(moreResults);
3205       return builder.build();
3206     } catch (IOException ie) {
3207       if (scannerName != null && ie instanceof NotServingRegionException) {
3208         scanners.remove(scannerName);
3209       }
3210       throw new ServiceException(ie);
3211     }
3212   }
3213 
3214   private void addResults(final ScanResponse.Builder builder, final List<Result> results,
3215       final RpcController controller) {
3216     if (results == null || results.isEmpty()) return;
3217     if (isClientCellBlockSupport()) {
3218       for (Result res : results) {
3219         builder.addCellsPerResult(res.size());
3220       }
3221       ((PayloadCarryingRpcController)controller).
3222         setCellScanner(CellUtil.createCellScanner(results));
3223     } else {
3224       for (Result res: results) {
3225         ClientProtos.Result pbr = ProtobufUtil.toResult(res);
3226         builder.addResults(pbr);
3227       }
3228     }
3229   }
3230 
3231   /**
3232    * Atomically bulk load several HFiles into an open region
3233    * @return true if successful, false is failed but recoverably (no action)
3234    * @throws IOException if failed unrecoverably
3235    */
3236   @Override
3237   public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller,
3238       final BulkLoadHFileRequest request) throws ServiceException {
3239     try {
3240       checkOpen();
3241       requestCount.increment();
3242       HRegion region = getRegion(request.getRegion());
3243       List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
3244       for (FamilyPath familyPath: request.getFamilyPathList()) {
3245         familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(),
3246           familyPath.getPath()));
3247       }
3248       boolean bypass = false;
3249       if (region.getCoprocessorHost() != null) {
3250         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
3251       }
3252       boolean loaded = false;
3253       if (!bypass) {
3254         loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum());
3255       }
3256       if (region.getCoprocessorHost() != null) {
3257         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
3258       }
3259       BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
3260       builder.setLoaded(loaded);
3261       return builder.build();
3262     } catch (IOException ie) {
3263       throw new ServiceException(ie);
3264     }
3265   }
3266 
3267   @Override
3268   public CoprocessorServiceResponse execService(final RpcController controller,
3269       final CoprocessorServiceRequest request) throws ServiceException {
3270     try {
3271       checkOpen();
3272       requestCount.increment();
3273       HRegion region = getRegion(request.getRegion());
3274       // ignore the passed in controller (from the serialized call)
3275       ServerRpcController execController = new ServerRpcController();
3276       Message result = region.execService(execController, request.getCall());
3277       if (execController.getFailedOn() != null) {
3278         throw execController.getFailedOn();
3279       }
3280       CoprocessorServiceResponse.Builder builder =
3281           CoprocessorServiceResponse.newBuilder();
3282       builder.setRegion(RequestConverter.buildRegionSpecifier(
3283           RegionSpecifierType.REGION_NAME, region.getRegionName()));
3284       builder.setValue(
3285           builder.getValueBuilder().setName(result.getClass().getName())
3286               .setValue(result.toByteString()));
3287       return builder.build();
3288     } catch (IOException ie) {
3289       throw new ServiceException(ie);
3290     }
3291   }
3292 
3293   /**
3294    * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
3295    *
3296    * @param rpcc the RPC controller
3297    * @param request the multi request
3298    * @throws ServiceException
3299    */
3300   @Override
3301   public MultiResponse multi(final RpcController rpcc, final MultiRequest request)
3302   throws ServiceException {
3303     try {
3304       checkOpen();
3305     } catch (IOException ie) {
3306       throw new ServiceException(ie);
3307     }
3308 
3309     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
3310     // It is also the conduit via which we pass back data.
3311     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
3312     CellScanner cellScanner = controller != null ? controller.cellScanner(): null;
3313     if (controller != null) controller.setCellScanner(null);
3314 
3315     long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
3316 
3317     // this will contain all the cells that we need to return. It's created later, if needed.
3318     List<CellScannable> cellsToReturn = null;
3319     MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
3320     RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
3321 
3322     for (RegionAction regionAction : request.getRegionActionList()) {
3323       this.requestCount.add(regionAction.getActionCount());
3324       HRegion region;
3325       regionActionResultBuilder.clear();
3326       try {
3327         region = getRegion(regionAction.getRegion());
3328       } catch (IOException e) {
3329         regionActionResultBuilder.setException(ResponseConverter.buildException(e));
3330         responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
3331         continue;  // For this region it's a failure.
3332       }
3333 
3334       if (regionAction.hasAtomic() && regionAction.getAtomic()) {
3335         // How does this call happen?  It may need some work to play well w/ the surroundings.
3336         // Need to return an item per Action along w/ Action index.  TODO.
3337         try {
3338           mutateRows(region, regionAction.getActionList(), cellScanner);
3339         } catch (IOException e) {
3340           // As it's atomic, we may expect it's a global failure.
3341           regionActionResultBuilder.setException(ResponseConverter.buildException(e));
3342         }
3343       } else {
3344         // doNonAtomicRegionMutation manages the exception internally
3345         cellsToReturn = doNonAtomicRegionMutation(region, regionAction, cellScanner,
3346             regionActionResultBuilder, cellsToReturn, nonceGroup);
3347       }
3348       responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
3349     }
3350     // Load the controller with the Cells to return.
3351     if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
3352       controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
3353     }
3354     return responseBuilder.build();
3355   }
3356 
3357   /**
3358    * Run through the regionMutation <code>rm</code> and per Mutation, do the work, and then when
3359    * done, add an instance of a {@link ResultOrException} that corresponds to each Mutation.
3360    * @param region
3361    * @param actions
3362    * @param cellScanner
3363    * @param builder
3364    * @param cellsToReturn  Could be null. May be allocated in this method.  This is what this
3365    * method returns as a 'result'.
3366    * @return Return the <code>cellScanner</code> passed
3367    */
3368   private List<CellScannable> doNonAtomicRegionMutation(final HRegion region,
3369       final RegionAction actions, final CellScanner cellScanner,
3370       final RegionActionResult.Builder builder, List<CellScannable> cellsToReturn, long nonceGroup) {
3371     // Gather up CONTIGUOUS Puts and Deletes in this mutations List.  Idea is that rather than do
3372     // one at a time, we instead pass them in batch.  Be aware that the corresponding
3373     // ResultOrException instance that matches each Put or Delete is then added down in the
3374     // doBatchOp call.  We should be staying aligned though the Put and Delete are deferred/batched
3375     List<ClientProtos.Action> mutations = null;
3376     for (ClientProtos.Action action: actions.getActionList()) {
3377       ClientProtos.ResultOrException.Builder resultOrExceptionBuilder = null;
3378       try {
3379         Result r = null;
3380         if (action.hasGet()) {
3381           Get get = ProtobufUtil.toGet(action.getGet());
3382           r = region.get(get);
3383         } else if (action.hasMutation()) {
3384           MutationType type = action.getMutation().getMutateType();
3385           if (type != MutationType.PUT && type != MutationType.DELETE && mutations != null &&
3386               !mutations.isEmpty()) {
3387             // Flush out any Puts or Deletes already collected.
3388             doBatchOp(builder, region, mutations, cellScanner);
3389             mutations.clear();
3390           }
3391           switch (type) {
3392           case APPEND:
3393             r = append(region, action.getMutation(), cellScanner, nonceGroup);
3394             break;
3395           case INCREMENT:
3396             r = increment(region, action.getMutation(), cellScanner,  nonceGroup);
3397             break;
3398           case PUT:
3399           case DELETE:
3400             // Collect the individual mutations and apply in a batch
3401             if (mutations == null) {
3402               mutations = new ArrayList<ClientProtos.Action>(actions.getActionCount());
3403             }
3404             mutations.add(action);
3405             break;
3406           default:
3407             throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
3408           }
3409         } else {
3410           throw new HBaseIOException("Unexpected Action type");
3411         }
3412         if (r != null) {
3413           ClientProtos.Result pbResult = null;
3414           if (isClientCellBlockSupport()) {
3415             pbResult = ProtobufUtil.toResultNoData(r);
3416             //  Hard to guess the size here.  Just make a rough guess.
3417             if (cellsToReturn == null) cellsToReturn = new ArrayList<CellScannable>();
3418             cellsToReturn.add(r);
3419           } else {
3420             pbResult = ProtobufUtil.toResult(r);
3421           }
3422           resultOrExceptionBuilder =
3423             ClientProtos.ResultOrException.newBuilder().setResult(pbResult);
3424         }
3425         // Could get to here and there was no result and no exception.  Presumes we added
3426         // a Put or Delete to the collecting Mutations List for adding later.  In this
3427         // case the corresponding ResultOrException instance for the Put or Delete will be added
3428         // down in the doBatchOp method call rather than up here.
3429       } catch (IOException ie) {
3430         resultOrExceptionBuilder = ResultOrException.newBuilder().
3431           setException(ResponseConverter.buildException(ie));
3432       }
3433       if (resultOrExceptionBuilder != null) {
3434         // Propagate index.
3435         resultOrExceptionBuilder.setIndex(action.getIndex());
3436         builder.addResultOrException(resultOrExceptionBuilder.build());
3437       }
3438     }
3439     // Finish up any outstanding mutations
3440     if (mutations != null && !mutations.isEmpty()) {
3441       doBatchOp(builder, region, mutations, cellScanner);
3442     }
3443     return cellsToReturn;
3444   }
3445 
3446 // End Client methods
3447 // Start Admin methods
3448 
3449   @Override
3450   @QosPriority(priority=HConstants.HIGH_QOS)
3451   public GetRegionInfoResponse getRegionInfo(final RpcController controller,
3452       final GetRegionInfoRequest request) throws ServiceException {
3453     try {
3454       checkOpen();
3455       requestCount.increment();
3456       HRegion region = getRegion(request.getRegion());
3457       HRegionInfo info = region.getRegionInfo();
3458       GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
3459       builder.setRegionInfo(HRegionInfo.convert(info));
3460       if (request.hasCompactionState() && request.getCompactionState()) {
3461         builder.setCompactionState(region.getCompactionState());
3462       }
3463       builder.setIsRecovering(region.isRecovering());
3464       return builder.build();
3465     } catch (IOException ie) {
3466       throw new ServiceException(ie);
3467     }
3468   }
3469 
3470   @Override
3471   public GetStoreFileResponse getStoreFile(final RpcController controller,
3472       final GetStoreFileRequest request) throws ServiceException {
3473     try {
3474       checkOpen();
3475       HRegion region = getRegion(request.getRegion());
3476       requestCount.increment();
3477       Set<byte[]> columnFamilies;
3478       if (request.getFamilyCount() == 0) {
3479         columnFamilies = region.getStores().keySet();
3480       } else {
3481         columnFamilies = new TreeSet<byte[]>(Bytes.BYTES_RAWCOMPARATOR);
3482         for (ByteString cf: request.getFamilyList()) {
3483           columnFamilies.add(cf.toByteArray());
3484         }
3485       }
3486       int nCF = columnFamilies.size();
3487       List<String>  fileList = region.getStoreFileList(
3488         columnFamilies.toArray(new byte[nCF][]));
3489       GetStoreFileResponse.Builder builder = GetStoreFileResponse.newBuilder();
3490       builder.addAllStoreFile(fileList);
3491       return builder.build();
3492     } catch (IOException ie) {
3493       throw new ServiceException(ie);
3494     }
3495   }
3496 
3497   @Override
3498   @QosPriority(priority=HConstants.HIGH_QOS)
3499   public GetOnlineRegionResponse getOnlineRegion(final RpcController controller,
3500       final GetOnlineRegionRequest request) throws ServiceException {
3501     try {
3502       checkOpen();
3503       requestCount.increment();
3504       List<HRegionInfo> list = new ArrayList<HRegionInfo>(onlineRegions.size());
3505       for (HRegion region: this.onlineRegions.values()) {
3506         list.add(region.getRegionInfo());
3507       }
3508       Collections.sort(list);
3509       return ResponseConverter.buildGetOnlineRegionResponse(list);
3510     } catch (IOException ie) {
3511       throw new ServiceException(ie);
3512     }
3513   }
3514 
3515   // Region open/close direct RPCs
3516 
3517   /**
3518    * Open asynchronously a region or a set of regions on the region server.
3519    *
3520    * The opening is coordinated by ZooKeeper, and this method requires the znode to be created
3521    *  before being called. As a consequence, this method should be called only from the master.
3522    * <p>
3523    * Different manages states for the region are:<ul>
3524    *  <li>region not opened: the region opening will start asynchronously.</li>
3525    *  <li>a close is already in progress: this is considered as an error.</li>
3526    *  <li>an open is already in progress: this new open request will be ignored. This is important
3527    *  because the Master can do multiple requests if it crashes.</li>
3528    *  <li>the region is already opened:  this new open request will be ignored./li>
3529    *  </ul>
3530    * </p>
3531    * <p>
3532    * Bulk assign: If there are more than 1 region to open, it will be considered as a bulk assign.
3533    * For a single region opening, errors are sent through a ServiceException. For bulk assign,
3534    * errors are put in the response as FAILED_OPENING.
3535    * </p>
3536    * @param controller the RPC controller
3537    * @param request the request
3538    * @throws ServiceException
3539    */
3540   @Override
3541   @QosPriority(priority=HConstants.HIGH_QOS)
3542   public OpenRegionResponse openRegion(final RpcController controller,
3543       final OpenRegionRequest request) throws ServiceException {
3544     try {
3545       checkOpen();
3546     } catch (IOException ie) {
3547       throw new ServiceException(ie);
3548     }
3549     requestCount.increment();
3550     OpenRegionResponse.Builder builder = OpenRegionResponse.newBuilder();
3551     final int regionCount = request.getOpenInfoCount();
3552     final Map<TableName, HTableDescriptor> htds =
3553         new HashMap<TableName, HTableDescriptor>(regionCount);
3554     final boolean isBulkAssign = regionCount > 1;
3555     for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
3556       final HRegionInfo region = HRegionInfo.convert(regionOpenInfo.getRegion());
3557 
3558       int versionOfOfflineNode = -1;
3559       if (regionOpenInfo.hasVersionOfOfflineNode()) {
3560         versionOfOfflineNode = regionOpenInfo.getVersionOfOfflineNode();
3561       }
3562       HTableDescriptor htd;
3563       try {
3564         final HRegion onlineRegion = getFromOnlineRegions(region.getEncodedName());
3565         if (onlineRegion != null) {
3566           //Check if the region can actually be opened.
3567           if (onlineRegion.getCoprocessorHost() != null) {
3568             onlineRegion.getCoprocessorHost().preOpen();
3569           }
3570           // See HBASE-5094. Cross check with hbase:meta if still this RS is owning
3571           // the region.
3572           Pair<HRegionInfo, ServerName> p = MetaReader.getRegion(
3573               this.catalogTracker, region.getRegionName());
3574           if (this.getServerName().equals(p.getSecond())) {
3575             Boolean closing = regionsInTransitionInRS.get(region.getEncodedNameAsBytes());
3576             // Map regionsInTransitionInRSOnly has an entry for a region only if the region
3577             // is in transition on this RS, so here closing can be null. If not null, it can
3578             // be true or false. True means the region is opening on this RS; while false
3579             // means the region is closing. Only return ALREADY_OPENED if not closing (i.e.
3580             // not in transition any more, or still transition to open.
3581             if (!Boolean.FALSE.equals(closing)
3582                 && getFromOnlineRegions(region.getEncodedName()) != null) {
3583               LOG.warn("Attempted open of " + region.getEncodedName()
3584                 + " but already online on this server");
3585               builder.addOpeningState(RegionOpeningState.ALREADY_OPENED);
3586               continue;
3587             }
3588           } else {
3589             LOG.warn("The region " + region.getEncodedName() + " is online on this server" +
3590                 " but hbase:meta does not have this server - continue opening.");
3591             removeFromOnlineRegions(onlineRegion, null);
3592           }
3593         }
3594         LOG.info("Open " + region.getRegionNameAsString());
3595         htd = htds.get(region.getTable());
3596         if (htd == null) {
3597           htd = this.tableDescriptors.get(region.getTable());
3598           htds.put(region.getTable(), htd);
3599         }
3600 
3601         final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(
3602             region.getEncodedNameAsBytes(), Boolean.TRUE);
3603 
3604         if (Boolean.FALSE.equals(previous)) {
3605           // There is a close in progress. We need to mark this open as failed in ZK.
3606           OpenRegionHandler.
3607               tryTransitionFromOfflineToFailedOpen(this, region, versionOfOfflineNode);
3608 
3609           throw new RegionAlreadyInTransitionException("Received OPEN for the region:" +
3610               region.getRegionNameAsString() + " , which we are already trying to CLOSE ");
3611         }
3612 
3613         if (Boolean.TRUE.equals(previous)) {
3614           // An open is in progress. This is supported, but let's log this.
3615           LOG.info("Receiving OPEN for the region:" +
3616               region.getRegionNameAsString() + " , which we are already trying to OPEN" +
3617               " - ignoring this new request for this region.");
3618         }
3619 
3620         // We are opening this region. If it moves back and forth for whatever reason, we don't
3621         // want to keep returning the stale moved record while we are opening/if we close again.
3622         removeFromMovedRegions(region.getEncodedName());
3623 
3624         if (previous == null) {
3625           // check if the region to be opened is marked in recovering state in ZK
3626           if (this.distributedLogReplay
3627               && SplitLogManager.isRegionMarkedRecoveringInZK(this.getZooKeeper(),
3628             region.getEncodedName())) {
3629             this.recoveringRegions.put(region.getEncodedName(), null);
3630           }
3631           // If there is no action in progress, we can submit a specific handler.
3632           // Need to pass the expected version in the constructor.
3633           if (region.isMetaRegion()) {
3634             this.service.submit(new OpenMetaHandler(this, this, region, htd,
3635                 versionOfOfflineNode));
3636           } else {
3637             updateRegionFavoredNodesMapping(region.getEncodedName(),
3638                 regionOpenInfo.getFavoredNodesList());
3639             this.service.submit(new OpenRegionHandler(this, this, region, htd,
3640                 versionOfOfflineNode));
3641           }
3642         }
3643 
3644         builder.addOpeningState(RegionOpeningState.OPENED);
3645 
3646       } catch (KeeperException zooKeeperEx) {
3647         LOG.error("Can't retrieve recovering state from zookeeper", zooKeeperEx);
3648         throw new ServiceException(zooKeeperEx);
3649       } catch (IOException ie) {
3650         LOG.warn("Failed opening region " + region.getRegionNameAsString(), ie);
3651         if (isBulkAssign) {
3652           builder.addOpeningState(RegionOpeningState.FAILED_OPENING);
3653         } else {
3654           throw new ServiceException(ie);
3655         }
3656       }
3657     }
3658 
3659     return builder.build();
3660   }
3661 
3662   @Override
3663   public void updateRegionFavoredNodesMapping(String encodedRegionName,
3664       List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) {
3665     InetSocketAddress[] addr = new InetSocketAddress[favoredNodes.size()];
3666     // Refer to the comment on the declaration of regionFavoredNodesMap on why
3667     // it is a map of region name to InetSocketAddress[]
3668     for (int i = 0; i < favoredNodes.size(); i++) {
3669       addr[i] = InetSocketAddress.createUnresolved(favoredNodes.get(i).getHostName(),
3670           favoredNodes.get(i).getPort());
3671     }
3672     regionFavoredNodesMap.put(encodedRegionName, addr);
3673   }
3674 
3675   /**
3676    * Return the favored nodes for a region given its encoded name. Look at the
3677    * comment around {@link #regionFavoredNodesMap} on why it is InetSocketAddress[]
3678    * @param encodedRegionName
3679    * @return array of favored locations
3680    */
3681   @Override
3682   public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
3683     return regionFavoredNodesMap.get(encodedRegionName);
3684   }
3685 
3686   /**
3687    * Close a region on the region server.
3688    *
3689    * @param controller the RPC controller
3690    * @param request the request
3691    * @throws ServiceException
3692    */
3693   @Override
3694   @QosPriority(priority=HConstants.HIGH_QOS)
3695   public CloseRegionResponse closeRegion(final RpcController controller,
3696       final CloseRegionRequest request) throws ServiceException {
3697     int versionOfClosingNode = -1;
3698     if (request.hasVersionOfClosingNode()) {
3699       versionOfClosingNode = request.getVersionOfClosingNode();
3700     }
3701     boolean zk = request.getTransitionInZK();
3702     final ServerName sn = (request.hasDestinationServer() ?
3703       ProtobufUtil.toServerName(request.getDestinationServer()) : null);
3704 
3705     try {
3706       checkOpen();
3707       final String encodedRegionName = ProtobufUtil.getRegionEncodedName(request.getRegion());
3708 
3709       // Can be null if we're calling close on a region that's not online
3710       final HRegion region = this.getFromOnlineRegions(encodedRegionName);
3711       if ((region  != null) && (region .getCoprocessorHost() != null)) {
3712         region.getCoprocessorHost().preClose(false);
3713       }
3714 
3715       requestCount.increment();
3716       LOG.info("Close " + encodedRegionName + ", via zk=" + (zk ? "yes" : "no") +
3717         ", znode version=" + versionOfClosingNode + ", on " + sn);
3718 
3719       boolean closed = closeRegion(encodedRegionName, false, zk, versionOfClosingNode, sn);
3720       CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder().setClosed(closed);
3721       return builder.build();
3722     } catch (IOException ie) {
3723       throw new ServiceException(ie);
3724     }
3725   }
3726 
3727   /**
3728    * Flush a region on the region server.
3729    *
3730    * @param controller the RPC controller
3731    * @param request the request
3732    * @throws ServiceException
3733    */
3734   @Override
3735   @QosPriority(priority=HConstants.HIGH_QOS)
3736   public FlushRegionResponse flushRegion(final RpcController controller,
3737       final FlushRegionRequest request) throws ServiceException {
3738     try {
3739       checkOpen();
3740       requestCount.increment();
3741       HRegion region = getRegion(request.getRegion());
3742       LOG.info("Flushing " + region.getRegionNameAsString());
3743       boolean shouldFlush = true;
3744       if (request.hasIfOlderThanTs()) {
3745         shouldFlush = region.getLastFlushTime() < request.getIfOlderThanTs();
3746       }
3747       FlushRegionResponse.Builder builder = FlushRegionResponse.newBuilder();
3748       if (shouldFlush) {
3749         boolean result = region.flushcache();
3750         if (result) {
3751           this.compactSplitThread.requestSystemCompaction(region,
3752               "Compaction through user triggered flush");
3753         }
3754         builder.setFlushed(result);
3755       }
3756       builder.setLastFlushTime(region.getLastFlushTime());
3757       return builder.build();
3758     } catch (IOException ie) {
3759       throw new ServiceException(ie);
3760     }
3761   }
3762 
3763   /**
3764    * Split a region on the region server.
3765    *
3766    * @param controller the RPC controller
3767    * @param request the request
3768    * @throws ServiceException
3769    */
3770   @Override
3771   @QosPriority(priority=HConstants.HIGH_QOS)
3772   public SplitRegionResponse splitRegion(final RpcController controller,
3773       final SplitRegionRequest request) throws ServiceException {
3774     try {
3775       checkOpen();
3776       requestCount.increment();
3777       HRegion region = getRegion(request.getRegion());
3778       region.startRegionOperation(Operation.SPLIT_REGION);
3779       LOG.info("Splitting " + region.getRegionNameAsString());
3780       region.flushcache();
3781       byte[] splitPoint = null;
3782       if (request.hasSplitPoint()) {
3783         splitPoint = request.getSplitPoint().toByteArray();
3784       }
3785       region.forceSplit(splitPoint);
3786       compactSplitThread.requestSplit(region, region.checkSplit());
3787       return SplitRegionResponse.newBuilder().build();
3788     } catch (IOException ie) {
3789       throw new ServiceException(ie);
3790     }
3791   }
3792 
3793   /**
3794    * Merge regions on the region server.
3795    *
3796    * @param controller the RPC controller
3797    * @param request the request
3798    * @return merge regions response
3799    * @throws ServiceException
3800    */
3801   @Override
3802   @QosPriority(priority = HConstants.HIGH_QOS)
3803   public MergeRegionsResponse mergeRegions(final RpcController controller,
3804       final MergeRegionsRequest request) throws ServiceException {
3805     try {
3806       checkOpen();
3807       requestCount.increment();
3808       HRegion regionA = getRegion(request.getRegionA());
3809       HRegion regionB = getRegion(request.getRegionB());
3810       boolean forcible = request.getForcible();
3811       regionA.startRegionOperation(Operation.MERGE_REGION);
3812       regionB.startRegionOperation(Operation.MERGE_REGION);
3813       LOG.info("Receiving merging request for  " + regionA + ", " + regionB
3814           + ",forcible=" + forcible);
3815       regionA.flushcache();
3816       regionB.flushcache();
3817       compactSplitThread.requestRegionsMerge(regionA, regionB, forcible);
3818       return MergeRegionsResponse.newBuilder().build();
3819     } catch (IOException ie) {
3820       throw new ServiceException(ie);
3821     }
3822   }
3823 
3824   /**
3825    * Compact a region on the region server.
3826    *
3827    * @param controller the RPC controller
3828    * @param request the request
3829    * @throws ServiceException
3830    */
3831   @Override
3832   @QosPriority(priority=HConstants.HIGH_QOS)
3833   public CompactRegionResponse compactRegion(final RpcController controller,
3834       final CompactRegionRequest request) throws ServiceException {
3835     try {
3836       checkOpen();
3837       requestCount.increment();
3838       HRegion region = getRegion(request.getRegion());
3839       region.startRegionOperation(Operation.COMPACT_REGION);
3840       LOG.info("Compacting " + region.getRegionNameAsString());
3841       boolean major = false;
3842       byte [] family = null;
3843       Store store = null;
3844       if (request.hasFamily()) {
3845         family = request.getFamily().toByteArray();
3846         store = region.getStore(family);
3847         if (store == null) {
3848           throw new ServiceException(new IOException("column family " + Bytes.toString(family) +
3849             " does not exist in region " + region.getRegionNameAsString()));
3850         }
3851       }
3852       if (request.hasMajor()) {
3853         major = request.getMajor();
3854       }
3855       if (major) {
3856         if (family != null) {
3857           store.triggerMajorCompaction();
3858         } else {
3859           region.triggerMajorCompaction();
3860         }
3861       }
3862 
3863       String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
3864       LOG.trace("User-triggered compaction requested for region " +
3865         region.getRegionNameAsString() + familyLogMsg);
3866       String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
3867       if(family != null) {
3868         compactSplitThread.requestCompaction(region, store, log,
3869           Store.PRIORITY_USER, null);
3870       } else {
3871         compactSplitThread.requestCompaction(region, log,
3872           Store.PRIORITY_USER, null);
3873       }
3874       return CompactRegionResponse.newBuilder().build();
3875     } catch (IOException ie) {
3876       throw new ServiceException(ie);
3877     }
3878   }
3879 
3880   /**
3881    * Replicate WAL entries on the region server.
3882    *
3883    * @param controller the RPC controller
3884    * @param request the request
3885    * @throws ServiceException
3886    */
3887   @Override
3888   @QosPriority(priority=HConstants.REPLICATION_QOS)
3889   public ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
3890       final ReplicateWALEntryRequest request)
3891   throws ServiceException {
3892     try {
3893       if (replicationSinkHandler != null) {
3894         checkOpen();
3895         requestCount.increment();
3896         this.replicationSinkHandler.replicateLogEntries(request.getEntryList(),
3897           ((PayloadCarryingRpcController)controller).cellScanner());
3898       }
3899       return ReplicateWALEntryResponse.newBuilder().build();
3900     } catch (IOException ie) {
3901       throw new ServiceException(ie);
3902     }
3903   }
3904 
3905   /**
3906    * Replay the given changes when distributedLogReplay WAL edits from a failed RS. The guarantee is
3907    * that the given mutations will be durable on the receiving RS if this method returns without any
3908    * exception.
3909    * @param controller the RPC controller
3910    * @param request the request
3911    * @throws ServiceException
3912    */
3913   @Override
3914   @QosPriority(priority = HConstants.REPLAY_QOS)
3915   public ReplicateWALEntryResponse replay(final RpcController controller,
3916       final ReplicateWALEntryRequest request) throws ServiceException {
3917     long before = EnvironmentEdgeManager.currentTimeMillis();
3918     CellScanner cells = ((PayloadCarryingRpcController) controller).cellScanner();
3919     try {
3920       checkOpen();
3921       List<WALEntry> entries = request.getEntryList();
3922       if (entries == null || entries.isEmpty()) {
3923         // empty input
3924         return ReplicateWALEntryResponse.newBuilder().build();
3925       }
3926       HRegion region = this.getRegionByEncodedName(
3927         entries.get(0).getKey().getEncodedRegionName().toStringUtf8());
3928       RegionCoprocessorHost coprocessorHost = region.getCoprocessorHost();
3929       List<Pair<HLogKey, WALEdit>> walEntries = new ArrayList<Pair<HLogKey, WALEdit>>();
3930       List<HLogSplitter.MutationReplay> mutations = new ArrayList<HLogSplitter.MutationReplay>();
3931       // when tag is enabled, we need tag replay edits with log sequence number
3932       boolean needAddReplayTag = (HFile.getFormatVersion(this.conf) >= 3);
3933       for (WALEntry entry : entries) {
3934         if (nonceManager != null) {
3935           long nonceGroup = entry.getKey().hasNonceGroup()
3936               ? entry.getKey().getNonceGroup() : HConstants.NO_NONCE;
3937           long nonce = entry.getKey().hasNonce() ? entry.getKey().getNonce() : HConstants.NO_NONCE;
3938           nonceManager.reportOperationFromWal(nonceGroup, nonce, entry.getKey().getWriteTime());
3939         }
3940         Pair<HLogKey, WALEdit> walEntry = (coprocessorHost == null) ? null :
3941           new Pair<HLogKey, WALEdit>();
3942         List<HLogSplitter.MutationReplay> edits = HLogSplitter.getMutationsFromWALEntry(entry,
3943           cells, walEntry, needAddReplayTag);
3944         if (coprocessorHost != null) {
3945           // Start coprocessor replay here. The coprocessor is for each WALEdit instead of a
3946           // KeyValue.
3947           if (coprocessorHost.preWALRestore(region.getRegionInfo(), walEntry.getFirst(),
3948             walEntry.getSecond())) {
3949             // if bypass this log entry, ignore it ...
3950             continue;
3951           }
3952           walEntries.add(walEntry);
3953         }
3954         mutations.addAll(edits);
3955       }
3956 
3957       if (!mutations.isEmpty()) {
3958         OperationStatus[] result = doReplayBatchOp(region, mutations);
3959         // check if it's a partial success
3960         for (int i = 0; result != null && i < result.length; i++) {
3961           if (result[i] != OperationStatus.SUCCESS) {
3962             throw new IOException(result[i].getExceptionMsg());
3963           }
3964         }
3965       }
3966       if (coprocessorHost != null) {
3967         for (Pair<HLogKey, WALEdit> wal : walEntries) {
3968           coprocessorHost.postWALRestore(region.getRegionInfo(), wal.getFirst(),
3969             wal.getSecond());
3970         }
3971       }
3972       return ReplicateWALEntryResponse.newBuilder().build();
3973     } catch (IOException ie) {
3974       throw new ServiceException(ie);
3975     } finally {
3976       metricsRegionServer.updateReplay(EnvironmentEdgeManager.currentTimeMillis() - before);
3977     }
3978   }
3979 
3980   /**
3981    * Roll the WAL writer of the region server.
3982    * @param controller the RPC controller
3983    * @param request the request
3984    * @throws ServiceException
3985    */
3986   @Override
3987   public RollWALWriterResponse rollWALWriter(final RpcController controller,
3988       final RollWALWriterRequest request) throws ServiceException {
3989     try {
3990       checkOpen();
3991       requestCount.increment();
3992       HLog wal = this.getWAL();
3993       byte[][] regionsToFlush = wal.rollWriter(true);
3994       RollWALWriterResponse.Builder builder = RollWALWriterResponse.newBuilder();
3995       if (regionsToFlush != null) {
3996         for (byte[] region: regionsToFlush) {
3997           builder.addRegionToFlush(HBaseZeroCopyByteString.wrap(region));
3998         }
3999       }
4000       return builder.build();
4001     } catch (IOException ie) {
4002       throw new ServiceException(ie);
4003     }
4004   }
4005 
4006   /**
4007    * Stop the region server.
4008    *
4009    * @param controller the RPC controller
4010    * @param request the request
4011    * @throws ServiceException
4012    */
4013   @Override
4014   public StopServerResponse stopServer(final RpcController controller,
4015       final StopServerRequest request) throws ServiceException {
4016     requestCount.increment();
4017     String reason = request.getReason();
4018     stop(reason);
4019     return StopServerResponse.newBuilder().build();
4020   }
4021 
4022   /**
4023    * Get some information of the region server.
4024    *
4025    * @param controller the RPC controller
4026    * @param request the request
4027    * @throws ServiceException
4028    */
4029   @Override
4030   public GetServerInfoResponse getServerInfo(final RpcController controller,
4031       final GetServerInfoRequest request) throws ServiceException {
4032     try {
4033       checkOpen();
4034     } catch (IOException ie) {
4035       throw new ServiceException(ie);
4036     }
4037     ServerName serverName = getServerName();
4038     requestCount.increment();
4039     return ResponseConverter.buildGetServerInfoResponse(serverName, rsInfo.getInfoPort());
4040   }
4041 
4042 // End Admin methods
4043 
4044   /**
4045    * Find the HRegion based on a region specifier
4046    *
4047    * @param regionSpecifier the region specifier
4048    * @return the corresponding region
4049    * @throws IOException if the specifier is not null,
4050    *    but failed to find the region
4051    */
4052   protected HRegion getRegion(
4053       final RegionSpecifier regionSpecifier) throws IOException {
4054     return getRegionByEncodedName(regionSpecifier.getValue().toByteArray(),
4055         ProtobufUtil.getRegionEncodedName(regionSpecifier));
4056   }
4057 
4058   /**
4059    * Execute an append mutation.
4060    *
4061    * @param region
4062    * @param m
4063    * @param cellScanner
4064    * @return result to return to client if default operation should be
4065    * bypassed as indicated by RegionObserver, null otherwise
4066    * @throws IOException
4067    */
4068   protected Result append(final HRegion region,
4069       final MutationProto m, final CellScanner cellScanner, long nonceGroup) throws IOException {
4070     long before = EnvironmentEdgeManager.currentTimeMillis();
4071     Append append = ProtobufUtil.toAppend(m, cellScanner);
4072     Result r = null;
4073     if (region.getCoprocessorHost() != null) {
4074       r = region.getCoprocessorHost().preAppend(append);
4075     }
4076     if (r == null) {
4077       long nonce = startNonceOperation(m, nonceGroup);
4078       boolean success = false;
4079       try {
4080         r = region.append(append, nonceGroup, nonce);
4081         success = true;
4082       } finally {
4083         endNonceOperation(m, nonceGroup, success);
4084       }
4085       if (region.getCoprocessorHost() != null) {
4086         region.getCoprocessorHost().postAppend(append, r);
4087       }
4088     }
4089     metricsRegionServer.updateAppend(EnvironmentEdgeManager.currentTimeMillis() - before);
4090     return r;
4091   }
4092 
4093   /**
4094    * Execute an increment mutation.
4095    *
4096    * @param region
4097    * @param mutation
4098    * @return the Result
4099    * @throws IOException
4100    */
4101   protected Result increment(final HRegion region, final MutationProto mutation,
4102       final CellScanner cells, long nonceGroup) throws IOException {
4103     long before = EnvironmentEdgeManager.currentTimeMillis();
4104     Increment increment = ProtobufUtil.toIncrement(mutation, cells);
4105     Result r = null;
4106     if (region.getCoprocessorHost() != null) {
4107       r = region.getCoprocessorHost().preIncrement(increment);
4108     }
4109     if (r == null) {
4110       long nonce = startNonceOperation(mutation, nonceGroup);
4111       boolean success = false;
4112       try {
4113         r = region.increment(increment, nonceGroup, nonce);
4114         success = true;
4115       } finally {
4116         endNonceOperation(mutation, nonceGroup, success);
4117       }
4118       if (region.getCoprocessorHost() != null) {
4119         r = region.getCoprocessorHost().postIncrement(increment, r);
4120       }
4121     }
4122     metricsRegionServer.updateIncrement(EnvironmentEdgeManager.currentTimeMillis() - before);
4123     return r;
4124   }
4125 
4126   /**
4127    * Starts the nonce operation for a mutation, if needed.
4128    * @param mutation Mutation.
4129    * @param nonceGroup Nonce group from the request.
4130    * @returns Nonce used (can be NO_NONCE).
4131    */
4132   private long startNonceOperation(final MutationProto mutation, long nonceGroup)
4133       throws IOException, OperationConflictException {
4134     if (nonceManager == null || !mutation.hasNonce()) return HConstants.NO_NONCE;
4135     boolean canProceed = false;
4136     try {
4137       canProceed = nonceManager.startOperation(nonceGroup, mutation.getNonce(), this);
4138     } catch (InterruptedException ex) {
4139       throw new InterruptedIOException("Nonce start operation interrupted");
4140     }
4141     if (!canProceed) {
4142       // TODO: instead, we could convert append/increment to get w/mvcc
4143       String message = "The operation with nonce {" + nonceGroup + ", " + mutation.getNonce()
4144           + "} on row [" + Bytes.toString(mutation.getRow().toByteArray())
4145           + "] may have already completed";
4146       throw new OperationConflictException(message);
4147     }
4148     return mutation.getNonce();
4149   }
4150 
4151   /**
4152    * Ends nonce operation for a mutation, if needed.
4153    * @param mutation Mutation.
4154    * @param nonceGroup Nonce group from the request. Always 0 in initial implementation.
4155    * @param success Whether the operation for this nonce has succeeded.
4156    */
4157   private void endNonceOperation(final MutationProto mutation, long nonceGroup,
4158       boolean success) {
4159     if (nonceManager == null || !mutation.hasNonce()) return;
4160     nonceManager.endOperation(nonceGroup, mutation.getNonce(), success);
4161   }
4162 
4163   @Override
4164   public ServerNonceManager getNonceManager() {
4165     return this.nonceManager;
4166   }
4167 
4168   /**
4169    * Execute a list of Put/Delete mutations.
4170    *
4171    * @param builder
4172    * @param region
4173    * @param mutations
4174    */
4175   protected void doBatchOp(final RegionActionResult.Builder builder, final HRegion region,
4176       final List<ClientProtos.Action> mutations, final CellScanner cells) {
4177     Mutation[] mArray = new Mutation[mutations.size()];
4178     long before = EnvironmentEdgeManager.currentTimeMillis();
4179     boolean batchContainsPuts = false, batchContainsDelete = false;
4180     try {
4181       int i = 0;
4182       for (ClientProtos.Action action: mutations) {
4183         MutationProto m = action.getMutation();
4184         Mutation mutation;
4185         if (m.getMutateType() == MutationType.PUT) {
4186           mutation = ProtobufUtil.toPut(m, cells);
4187           batchContainsPuts = true;
4188         } else {
4189           mutation = ProtobufUtil.toDelete(m, cells);
4190           batchContainsDelete = true;
4191         }
4192         mArray[i++] = mutation;
4193       }
4194 
4195       requestCount.add(mutations.size());
4196       if (!region.getRegionInfo().isMetaTable()) {
4197         cacheFlusher.reclaimMemStoreMemory();
4198       }
4199 
4200       OperationStatus codes[] = region.batchMutate(mArray);
4201       for (i = 0; i < codes.length; i++) {
4202         int index = mutations.get(i).getIndex();
4203         Exception e = null;
4204         switch (codes[i].getOperationStatusCode()) {
4205           case BAD_FAMILY:
4206             e = new NoSuchColumnFamilyException(codes[i].getExceptionMsg());
4207             builder.addResultOrException(getResultOrException(e, index));
4208             break;
4209 
4210           case SANITY_CHECK_FAILURE:
4211             e = new FailedSanityCheckException(codes[i].getExceptionMsg());
4212             builder.addResultOrException(getResultOrException(e, index));
4213             break;
4214 
4215           default:
4216             e = new DoNotRetryIOException(codes[i].getExceptionMsg());
4217             builder.addResultOrException(getResultOrException(e, index));
4218             break;
4219 
4220           case SUCCESS:
4221             builder.addResultOrException(getResultOrException(ClientProtos.Result.getDefaultInstance(), index));
4222             break;
4223         }
4224       }
4225     } catch (IOException ie) {
4226       for (int i = 0; i < mutations.size(); i++) {
4227         builder.addResultOrException(getResultOrException(ie, mutations.get(i).getIndex()));
4228       }
4229     }
4230     long after = EnvironmentEdgeManager.currentTimeMillis();
4231     if (batchContainsPuts) {
4232       metricsRegionServer.updatePut(after - before);
4233     }
4234     if (batchContainsDelete) {
4235       metricsRegionServer.updateDelete(after - before);
4236     }
4237   }
4238   private static ResultOrException getResultOrException(final ClientProtos.Result r,
4239       final int index) {
4240     return getResultOrException(ResponseConverter.buildActionResult(r), index);
4241   }
4242   private static ResultOrException getResultOrException(final Exception e, final int index) {
4243     return getResultOrException(ResponseConverter.buildActionResult(e), index);
4244   }
4245 
4246   private static ResultOrException getResultOrException(final ResultOrException.Builder builder,
4247       final int index) {
4248     return builder.setIndex(index).build();
4249   }
4250 
4251   /**
4252    * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of
4253    * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse.
4254    * @param region
4255    * @param mutations
4256    * @return an array of OperationStatus which internally contains the OperationStatusCode and the
4257    *         exceptionMessage if any
4258    * @throws IOException
4259    */
4260   protected OperationStatus [] doReplayBatchOp(final HRegion region,
4261       final List<HLogSplitter.MutationReplay> mutations) throws IOException {
4262     HLogSplitter.MutationReplay[] mArray = new HLogSplitter.MutationReplay[mutations.size()];
4263 
4264     long before = EnvironmentEdgeManager.currentTimeMillis();
4265     boolean batchContainsPuts = false, batchContainsDelete = false;
4266     try {
4267       int i = 0;
4268       for (HLogSplitter.MutationReplay m : mutations) {
4269         if (m.type == MutationType.PUT) {
4270           batchContainsPuts = true;
4271         } else {
4272           batchContainsDelete = true;
4273         }
4274         mArray[i++] = m;
4275       }
4276       requestCount.add(mutations.size());
4277       if (!region.getRegionInfo().isMetaTable()) {
4278         cacheFlusher.reclaimMemStoreMemory();
4279       }
4280       return region.batchReplay(mArray);
4281     } finally {
4282       long after = EnvironmentEdgeManager.currentTimeMillis();
4283       if (batchContainsPuts) {
4284         metricsRegionServer.updatePut(after - before);
4285       }
4286       if (batchContainsDelete) {
4287         metricsRegionServer.updateDelete(after - before);
4288       }
4289     }
4290   }
4291 
4292   /**
4293    * Mutate a list of rows atomically.
4294    *
4295    * @param region
4296    * @param actions
4297  * @param cellScanner if non-null, the mutation data -- the Cell content.
4298    * @throws IOException
4299    */
4300   protected void mutateRows(final HRegion region, final List<ClientProtos.Action> actions,
4301       final CellScanner cellScanner)
4302   throws IOException {
4303     if (!region.getRegionInfo().isMetaTable()) {
4304       cacheFlusher.reclaimMemStoreMemory();
4305     }
4306     RowMutations rm = null;
4307     for (ClientProtos.Action action: actions) {
4308       if (action.hasGet()) {
4309         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
4310           action.getGet());
4311       }
4312       MutationType type = action.getMutation().getMutateType();
4313       if (rm == null) {
4314         rm = new RowMutations(action.getMutation().getRow().toByteArray());
4315       }
4316       switch (type) {
4317       case PUT:
4318         rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
4319         break;
4320       case DELETE:
4321         rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
4322         break;
4323         default:
4324           throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
4325       }
4326     }
4327     region.mutateRow(rm);
4328   }
4329 
4330   private static class MovedRegionInfo {
4331     private final ServerName serverName;
4332     private final long seqNum;
4333     private final long ts;
4334 
4335     public MovedRegionInfo(ServerName serverName, long closeSeqNum) {
4336       this.serverName = serverName;
4337       this.seqNum = closeSeqNum;
4338       ts = EnvironmentEdgeManager.currentTimeMillis();
4339      }
4340 
4341     public ServerName getServerName() {
4342       return serverName;
4343     }
4344 
4345     public long getSeqNum() {
4346       return seqNum;
4347     }
4348 
4349     public long getMoveTime() {
4350       return ts;
4351     }
4352   }
4353 
4354   // This map will contains all the regions that we closed for a move.
4355   //  We add the time it was moved as we don't want to keep too old information
4356   protected Map<String, MovedRegionInfo> movedRegions =
4357       new ConcurrentHashMap<String, MovedRegionInfo>(3000);
4358 
4359   // We need a timeout. If not there is a risk of giving a wrong information: this would double
4360   //  the number of network calls instead of reducing them.
4361   private static final int TIMEOUT_REGION_MOVED = (2 * 60 * 1000);
4362 
4363   protected void addToMovedRegions(String encodedName, ServerName destination, long closeSeqNum) {
4364     if (ServerName.isSameHostnameAndPort(destination, this.getServerName())) {
4365       LOG.warn("Not adding moved region record: " + encodedName + " to self.");
4366       return;
4367     }
4368     LOG.info("Adding moved region record: " + encodedName + " to "
4369         + destination.getServerName() + ":" + destination.getPort()
4370         + " as of " + closeSeqNum);
4371     movedRegions.put(encodedName, new MovedRegionInfo(destination, closeSeqNum));
4372   }
4373 
4374   private void removeFromMovedRegions(String encodedName) {
4375     movedRegions.remove(encodedName);
4376   }
4377 
4378   private MovedRegionInfo getMovedRegion(final String encodedRegionName) {
4379     MovedRegionInfo dest = movedRegions.get(encodedRegionName);
4380 
4381     long now = EnvironmentEdgeManager.currentTimeMillis();
4382     if (dest != null) {
4383       if (dest.getMoveTime() > (now - TIMEOUT_REGION_MOVED)) {
4384         return dest;
4385       } else {
4386         movedRegions.remove(encodedRegionName);
4387       }
4388     }
4389 
4390     return null;
4391   }
4392 
4393   /**
4394    * Remove the expired entries from the moved regions list.
4395    */
4396   protected void cleanMovedRegions() {
4397     final long cutOff = System.currentTimeMillis() - TIMEOUT_REGION_MOVED;
4398     Iterator<Entry<String, MovedRegionInfo>> it = movedRegions.entrySet().iterator();
4399 
4400     while (it.hasNext()){
4401       Map.Entry<String, MovedRegionInfo> e = it.next();
4402       if (e.getValue().getMoveTime() < cutOff) {
4403         it.remove();
4404       }
4405     }
4406   }
4407 
4408   /**
4409    * Creates a Chore thread to clean the moved region cache.
4410    */
4411   protected static class MovedRegionsCleaner extends Chore implements Stoppable {
4412     private HRegionServer regionServer;
4413     Stoppable stoppable;
4414 
4415     private MovedRegionsCleaner(
4416       HRegionServer regionServer, Stoppable stoppable){
4417       super("MovedRegionsCleaner for region "+regionServer, TIMEOUT_REGION_MOVED, stoppable);
4418       this.regionServer = regionServer;
4419       this.stoppable = stoppable;
4420     }
4421 
4422     static MovedRegionsCleaner createAndStart(HRegionServer rs){
4423       Stoppable stoppable = new Stoppable() {
4424         private volatile boolean isStopped = false;
4425         @Override public void stop(String why) { isStopped = true;}
4426         @Override public boolean isStopped() {return isStopped;}
4427       };
4428 
4429       return new MovedRegionsCleaner(rs, stoppable);
4430     }
4431 
4432     @Override
4433     protected void chore() {
4434       regionServer.cleanMovedRegions();
4435     }
4436 
4437     @Override
4438     public void stop(String why) {
4439       stoppable.stop(why);
4440     }
4441 
4442     @Override
4443     public boolean isStopped() {
4444       return stoppable.isStopped();
4445     }
4446   }
4447 
4448   private String getMyEphemeralNodePath() {
4449     return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
4450   }
4451 
4452   /**
4453    * Holder class which holds the RegionScanner and nextCallSeq together.
4454    */
4455   private static class RegionScannerHolder {
4456     private RegionScanner s;
4457     private long nextCallSeq = 0L;
4458     private HRegion r;
4459 
4460     public RegionScannerHolder(RegionScanner s, HRegion r) {
4461       this.s = s;
4462       this.r = r;
4463     }
4464   }
4465 
4466   private boolean isHealthCheckerConfigured() {
4467     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
4468     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
4469   }
4470 
4471   /**
4472    * @return the underlying {@link CompactSplitThread} for the servers
4473    */
4474   public CompactSplitThread getCompactSplitThread() {
4475     return this.compactSplitThread;
4476   }
4477 
4478   /**
4479    * A helper function to store the last flushed sequence Id with the previous failed RS for a
4480    * recovering region. The Id is used to skip wal edits which are flushed. Since the flushed
4481    * sequence id is only valid for each RS, we associate the Id with corresponding failed RS.
4482    * @throws KeeperException
4483    * @throws IOException
4484    */
4485   private void updateRecoveringRegionLastFlushedSequenceId(HRegion r) throws KeeperException,
4486       IOException {
4487     if (!r.isRecovering()) {
4488       // return immdiately for non-recovering regions
4489       return;
4490     }
4491 
4492     HRegionInfo region = r.getRegionInfo();
4493     ZooKeeperWatcher zkw = getZooKeeper();
4494     String previousRSName = this.getLastFailedRSFromZK(region.getEncodedName());
4495     Map<byte[], Long> maxSeqIdInStores = r.getMaxStoreSeqIdForLogReplay();
4496     long minSeqIdForLogReplay = -1;
4497     for (Long storeSeqIdForReplay : maxSeqIdInStores.values()) {
4498       if (minSeqIdForLogReplay == -1 || storeSeqIdForReplay < minSeqIdForLogReplay) {
4499         minSeqIdForLogReplay = storeSeqIdForReplay;
4500       }
4501     }
4502     long lastRecordedFlushedSequenceId = -1;
4503     String nodePath = ZKUtil.joinZNode(this.zooKeeper.recoveringRegionsZNode,
4504       region.getEncodedName());
4505     // recovering-region level
4506     byte[] data = ZKUtil.getData(zkw, nodePath);
4507     if (data != null) {
4508       lastRecordedFlushedSequenceId = SplitLogManager.parseLastFlushedSequenceIdFrom(data);
4509     }
4510     if (data == null || lastRecordedFlushedSequenceId < minSeqIdForLogReplay) {
4511       ZKUtil.setData(zkw, nodePath, ZKUtil.positionToByteArray(minSeqIdForLogReplay));
4512     }
4513     if (previousRSName != null) {
4514       // one level deeper for the failed RS
4515       nodePath = ZKUtil.joinZNode(nodePath, previousRSName);
4516       ZKUtil.setData(zkw, nodePath,
4517         ZKUtil.regionSequenceIdsToByteArray(minSeqIdForLogReplay, maxSeqIdInStores));
4518       LOG.debug("Update last flushed sequence id of region " + region.getEncodedName() + " for "
4519           + previousRSName);
4520     } else {
4521       LOG.warn("Can't find failed region server for recovering region " + region.getEncodedName());
4522     }
4523   }
4524 
4525   /**
4526    * Return the last failed RS name under /hbase/recovering-regions/encodedRegionName
4527    * @param encodedRegionName
4528    * @throws KeeperException
4529    */
4530   private String getLastFailedRSFromZK(String encodedRegionName) throws KeeperException {
4531     String result = null;
4532     long maxZxid = 0;
4533     ZooKeeperWatcher zkw = this.getZooKeeper();
4534     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
4535     List<String> failedServers = ZKUtil.listChildrenNoWatch(zkw, nodePath);
4536     if (failedServers == null || failedServers.isEmpty()) {
4537       return result;
4538     }
4539     for (String failedServer : failedServers) {
4540       String rsPath = ZKUtil.joinZNode(nodePath, failedServer);
4541       Stat stat = new Stat();
4542       ZKUtil.getDataNoWatch(zkw, rsPath, stat);
4543       if (maxZxid < stat.getCzxid()) {
4544         maxZxid = stat.getCzxid();
4545         result = failedServer;
4546       }
4547     }
4548     return result;
4549   }
4550 
4551   @Override
4552   public UpdateFavoredNodesResponse updateFavoredNodes(RpcController controller,
4553       UpdateFavoredNodesRequest request) throws ServiceException {
4554     List<UpdateFavoredNodesRequest.RegionUpdateInfo> openInfoList = request.getUpdateInfoList();
4555     UpdateFavoredNodesResponse.Builder respBuilder = UpdateFavoredNodesResponse.newBuilder();
4556     for (UpdateFavoredNodesRequest.RegionUpdateInfo regionUpdateInfo : openInfoList) {
4557       HRegionInfo hri = HRegionInfo.convert(regionUpdateInfo.getRegion());
4558       updateRegionFavoredNodesMapping(hri.getEncodedName(),
4559           regionUpdateInfo.getFavoredNodesList());
4560     }
4561     respBuilder.setResponse(openInfoList.size());
4562     return respBuilder.build();
4563   }
4564 }