View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distr=ibuted on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.lang.reflect.Constructor;
26  import java.lang.reflect.InvocationTargetException;
27  import java.lang.reflect.Method;
28  import java.lang.reflect.ParameterizedType;
29  import java.lang.reflect.Type;
30  import java.nio.ByteBuffer;
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.NavigableSet;
38  
39  import com.google.protobuf.HBaseZeroCopyByteString;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.CellScanner;
44  import org.apache.hadoop.hbase.CellUtil;
45  import org.apache.hadoop.hbase.DoNotRetryIOException;
46  import org.apache.hadoop.hbase.HBaseConfiguration;
47  import org.apache.hadoop.hbase.HConstants;
48  import org.apache.hadoop.hbase.HRegionInfo;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.KeyValueUtil;
52  import org.apache.hadoop.hbase.NamespaceDescriptor;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableName;
55  import org.apache.hadoop.hbase.Tag;
56  import org.apache.hadoop.hbase.client.Append;
57  import org.apache.hadoop.hbase.client.Delete;
58  import org.apache.hadoop.hbase.client.Durability;
59  import org.apache.hadoop.hbase.client.Get;
60  import org.apache.hadoop.hbase.client.Increment;
61  import org.apache.hadoop.hbase.client.Mutation;
62  import org.apache.hadoop.hbase.client.Put;
63  import org.apache.hadoop.hbase.client.Result;
64  import org.apache.hadoop.hbase.client.Scan;
65  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
66  import org.apache.hadoop.hbase.exceptions.DeserializationException;
67  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
68  import org.apache.hadoop.hbase.filter.Filter;
69  import org.apache.hadoop.hbase.io.TimeRange;
70  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
71  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
72  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
87  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
88  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
90  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
105 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
107 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
108 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
110 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
111 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
112 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
113 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
114 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
118 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
121 import org.apache.hadoop.hbase.security.access.Permission;
122 import org.apache.hadoop.hbase.security.access.TablePermission;
123 import org.apache.hadoop.hbase.security.access.UserPermission;
124 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
125 import org.apache.hadoop.hbase.security.visibility.Authorizations;
126 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
127 import org.apache.hadoop.hbase.util.Bytes;
128 import org.apache.hadoop.hbase.util.DynamicClassLoader;
129 import org.apache.hadoop.hbase.util.ExceptionUtil;
130 import org.apache.hadoop.hbase.util.Methods;
131 import org.apache.hadoop.hbase.util.Pair;
132 import org.apache.hadoop.io.Text;
133 import org.apache.hadoop.ipc.RemoteException;
134 import org.apache.hadoop.security.token.Token;
135 
136 import com.google.common.collect.ArrayListMultimap;
137 import com.google.common.collect.ListMultimap;
138 import com.google.common.collect.Lists;
139 import com.google.protobuf.ByteString;
140 import com.google.protobuf.InvalidProtocolBufferException;
141 import com.google.protobuf.Message;
142 import com.google.protobuf.Parser;
143 import com.google.protobuf.RpcChannel;
144 import com.google.protobuf.Service;
145 import com.google.protobuf.ServiceException;
146 import com.google.protobuf.TextFormat;
147 
148 /**
149  * Protobufs utility.
150  */
151 public final class ProtobufUtil {
152 
153   private ProtobufUtil() {
154   }
155 
156   /**
157    * Primitive type to class mapping.
158    */
159   private final static Map<String, Class<?>>
160     PRIMITIVES = new HashMap<String, Class<?>>();
161 
162 
163   /**
164    * Many results are simple: no cell, exists true or false. To save on object creations,
165    *  we reuse them across calls.
166    */
167   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
168   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
169   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
170   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
171 
172   private final static ClientProtos.Result EMPTY_RESULT_PB;
173   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
174   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
175 
176   static {
177     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
178 
179     builder.setExists(true);
180     builder.setAssociatedCellCount(0);
181     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
182 
183     builder.clear();
184 
185     builder.setExists(false);
186     builder.setAssociatedCellCount(0);
187     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
188 
189     builder.clear();
190     builder.setAssociatedCellCount(0);
191     EMPTY_RESULT_PB =  builder.build();
192   }
193 
194   /**
195    * Dynamic class loader to load filter/comparators
196    */
197   private final static ClassLoader CLASS_LOADER;
198 
199   static {
200     ClassLoader parent = ProtobufUtil.class.getClassLoader();
201     Configuration conf = HBaseConfiguration.create();
202     CLASS_LOADER = new DynamicClassLoader(conf, parent);
203 
204     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
205     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
206     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
207     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
208     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
209     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
210     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
211     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
212     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
213   }
214 
215   /**
216    * Magic we put ahead of a serialized protobuf message.
217    * For example, all znode content is protobuf messages with the below magic
218    * for preamble.
219    */
220   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
221   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
222 
223   /**
224    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
225    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
226    * @param bytes Bytes to decorate
227    * @return The passed <code>bytes</codes> with magic prepended (Creates a new
228    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
229    */
230   public static byte [] prependPBMagic(final byte [] bytes) {
231     return Bytes.add(PB_MAGIC, bytes);
232   }
233 
234   /**
235    * @param bytes Bytes to check.
236    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
237    */
238   public static boolean isPBMagicPrefix(final byte [] bytes) {
239     return isPBMagicPrefix(bytes, 0, bytes.length);
240   }
241 
242   /**
243    * @param bytes Bytes to check.
244    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
245    */
246   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
247     if (bytes == null || len < PB_MAGIC.length) return false;
248     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
249   }
250 
251   /**
252    * @param bytes
253    * @throws DeserializationException if we are missing the pb magic prefix
254    */
255   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
256     if (!isPBMagicPrefix(bytes)) {
257       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
258     }
259   }
260 
261   /**
262    * @return Length of {@link #PB_MAGIC}
263    */
264   public static int lengthOfPBMagic() {
265     return PB_MAGIC.length;
266   }
267 
268   /**
269    * Return the IOException thrown by the remote server wrapped in
270    * ServiceException as cause.
271    *
272    * @param se ServiceException that wraps IO exception thrown by the server
273    * @return Exception wrapped in ServiceException or
274    *   a new IOException that wraps the unexpected ServiceException.
275    */
276   public static IOException getRemoteException(ServiceException se) {
277     Throwable e = se.getCause();
278     if (e == null) {
279       return new IOException(se);
280     }
281     if (ExceptionUtil.isInterrupt(e)) {
282       return ExceptionUtil.asInterrupt(e);
283     }
284     if (e instanceof RemoteException) {
285       e = ((RemoteException) e).unwrapRemoteException();
286     }
287     return e instanceof IOException ? (IOException) e : new IOException(se);
288   }
289 
290   /**
291    * Convert a ServerName to a protocol buffer ServerName
292    *
293    * @param serverName the ServerName to convert
294    * @return the converted protocol buffer ServerName
295    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
296    */
297   public static HBaseProtos.ServerName
298       toServerName(final ServerName serverName) {
299     if (serverName == null) return null;
300     HBaseProtos.ServerName.Builder builder =
301       HBaseProtos.ServerName.newBuilder();
302     builder.setHostName(serverName.getHostname());
303     if (serverName.getPort() >= 0) {
304       builder.setPort(serverName.getPort());
305     }
306     if (serverName.getStartcode() >= 0) {
307       builder.setStartCode(serverName.getStartcode());
308     }
309     return builder.build();
310   }
311 
312   /**
313    * Convert a protocol buffer ServerName to a ServerName
314    *
315    * @param proto the protocol buffer ServerName to convert
316    * @return the converted ServerName
317    */
318   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
319     if (proto == null) return null;
320     String hostName = proto.getHostName();
321     long startCode = -1;
322     int port = -1;
323     if (proto.hasPort()) {
324       port = proto.getPort();
325     }
326     if (proto.hasStartCode()) {
327       startCode = proto.getStartCode();
328     }
329     return ServerName.valueOf(hostName, port, startCode);
330   }
331 
332   /**
333    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
334    *
335    * @param proto the GetTableDescriptorsResponse
336    * @return HTableDescriptor[]
337    */
338   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
339     if (proto == null) return null;
340 
341     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
342     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
343       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
344     }
345     return ret;
346   }
347 
348   /**
349    * get the split keys in form "byte [][]" from a CreateTableRequest proto
350    *
351    * @param proto the CreateTableRequest
352    * @return the split keys
353    */
354   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
355     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
356     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
357       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
358     }
359     return splitKeys;
360   }
361 
362   /**
363    * Convert a protobuf Durability into a client Durability
364    */
365   public static Durability toDurability(
366       final ClientProtos.MutationProto.Durability proto) {
367     switch(proto) {
368     case USE_DEFAULT:
369       return Durability.USE_DEFAULT;
370     case SKIP_WAL:
371       return Durability.SKIP_WAL;
372     case ASYNC_WAL:
373       return Durability.ASYNC_WAL;
374     case SYNC_WAL:
375       return Durability.SYNC_WAL;
376     case FSYNC_WAL:
377       return Durability.FSYNC_WAL;
378     default:
379       return Durability.USE_DEFAULT;
380     }
381   }
382 
383   /**
384    * Convert a client Durability into a protbuf Durability
385    */
386   public static ClientProtos.MutationProto.Durability toDurability(
387       final Durability d) {
388     switch(d) {
389     case USE_DEFAULT:
390       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
391     case SKIP_WAL:
392       return ClientProtos.MutationProto.Durability.SKIP_WAL;
393     case ASYNC_WAL:
394       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
395     case SYNC_WAL:
396       return ClientProtos.MutationProto.Durability.SYNC_WAL;
397     case FSYNC_WAL:
398       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
399     default:
400       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
401     }
402   }
403 
404   /**
405    * Convert a protocol buffer Get to a client Get
406    *
407    * @param proto the protocol buffer Get to convert
408    * @return the converted client Get
409    * @throws IOException
410    */
411   public static Get toGet(
412       final ClientProtos.Get proto) throws IOException {
413     if (proto == null) return null;
414     byte[] row = proto.getRow().toByteArray();
415     Get get = new Get(row);
416     if (proto.hasCacheBlocks()) {
417       get.setCacheBlocks(proto.getCacheBlocks());
418     }
419     if (proto.hasMaxVersions()) {
420       get.setMaxVersions(proto.getMaxVersions());
421     }
422     if (proto.hasStoreLimit()) {
423       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
424     }
425     if (proto.hasStoreOffset()) {
426       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
427     }
428     if (proto.hasTimeRange()) {
429       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
430       long minStamp = 0;
431       long maxStamp = Long.MAX_VALUE;
432       if (timeRange.hasFrom()) {
433         minStamp = timeRange.getFrom();
434       }
435       if (timeRange.hasTo()) {
436         maxStamp = timeRange.getTo();
437       }
438       get.setTimeRange(minStamp, maxStamp);
439     }
440     if (proto.hasFilter()) {
441       FilterProtos.Filter filter = proto.getFilter();
442       get.setFilter(ProtobufUtil.toFilter(filter));
443     }
444     for (NameBytesPair attribute: proto.getAttributeList()) {
445       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
446     }
447     if (proto.getColumnCount() > 0) {
448       for (Column column: proto.getColumnList()) {
449         byte[] family = column.getFamily().toByteArray();
450         if (column.getQualifierCount() > 0) {
451           for (ByteString qualifier: column.getQualifierList()) {
452             get.addColumn(family, qualifier.toByteArray());
453           }
454         } else {
455           get.addFamily(family);
456         }
457       }
458     }
459     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
460       get.setCheckExistenceOnly(true);
461     }
462     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
463       get.setClosestRowBefore(true);
464     }
465     return get;
466   }
467 
468   /**
469    * Convert a protocol buffer Mutate to a Put.
470    *
471    * @param proto The protocol buffer MutationProto to convert
472    * @return A client Put.
473    * @throws IOException
474    */
475   public static Put toPut(final MutationProto proto)
476   throws IOException {
477     return toPut(proto, null);
478   }
479 
480   /**
481    * Convert a protocol buffer Mutate to a Put.
482    *
483    * @param proto The protocol buffer MutationProto to convert
484    * @param cellScanner If non-null, the Cell data that goes with this proto.
485    * @return A client Put.
486    * @throws IOException
487    */
488   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
489   throws IOException {
490     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
491     MutationType type = proto.getMutateType();
492     assert type == MutationType.PUT: type.name();
493     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
494     Put put = null;
495     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
496     if (cellCount > 0) {
497       // The proto has metadata only and the data is separate to be found in the cellScanner.
498       if (cellScanner == null) {
499         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
500             toShortString(proto));
501       }
502       for (int i = 0; i < cellCount; i++) {
503         if (!cellScanner.advance()) {
504           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
505             " no cell returned: " + toShortString(proto));
506         }
507         Cell cell = cellScanner.current();
508         if (put == null) {
509           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
510         }
511         put.add(cell);
512       }
513     } else {
514       if (proto.hasRow()) {
515         put = new Put(proto.getRow().asReadOnlyByteBuffer(), timestamp);
516       } else {
517         throw new IllegalArgumentException("row cannot be null");
518       }
519       // The proto has the metadata and the data itself
520       for (ColumnValue column: proto.getColumnValueList()) {
521         byte[] family = column.getFamily().toByteArray();
522         for (QualifierValue qv: column.getQualifierValueList()) {
523           if (!qv.hasValue()) {
524             throw new DoNotRetryIOException(
525                 "Missing required field: qualifier value");
526           }
527           ByteBuffer qualifier =
528               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
529           ByteBuffer value =
530               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
531           long ts = timestamp;
532           if (qv.hasTimestamp()) {
533             ts = qv.getTimestamp();
534           }
535           byte[] tags;
536           if (qv.hasTags()) {
537             tags = qv.getTags().toByteArray();
538             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
539             Tag[] tagArray = new Tag[array.length];
540             for(int i = 0; i< array.length; i++) {
541               tagArray[i] = (Tag)array[i];
542             }
543             put.addImmutable(family, qualifier, ts, value, tagArray);
544           } else {
545             put.addImmutable(family, qualifier, ts, value);
546           }
547         }
548       }
549     }
550     put.setDurability(toDurability(proto.getDurability()));
551     for (NameBytesPair attribute: proto.getAttributeList()) {
552       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
553     }
554     return put;
555   }
556 
557   /**
558    * Convert a protocol buffer Mutate to a Delete
559    *
560    * @param proto the protocol buffer Mutate to convert
561    * @return the converted client Delete
562    * @throws IOException
563    */
564   public static Delete toDelete(final MutationProto proto)
565   throws IOException {
566     return toDelete(proto, null);
567   }
568 
569   /**
570    * Convert a protocol buffer Mutate to a Delete
571    *
572    * @param proto the protocol buffer Mutate to convert
573    * @param cellScanner if non-null, the data that goes with this delete.
574    * @return the converted client Delete
575    * @throws IOException
576    */
577   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
578   throws IOException {
579     MutationType type = proto.getMutateType();
580     assert type == MutationType.DELETE : type.name();
581     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
582     long timestamp = HConstants.LATEST_TIMESTAMP;
583     if (proto.hasTimestamp()) {
584       timestamp = proto.getTimestamp();
585     }
586     Delete delete = null;
587     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
588     if (cellCount > 0) {
589       // The proto has metadata only and the data is separate to be found in the cellScanner.
590       if (cellScanner == null) {
591         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
592         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
593           TextFormat.shortDebugString(proto));
594       }
595       for (int i = 0; i < cellCount; i++) {
596         if (!cellScanner.advance()) {
597           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
598           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
599             " no cell returned: " + TextFormat.shortDebugString(proto));
600         }
601         Cell cell = cellScanner.current();
602         if (delete == null) {
603           delete =
604             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
605         }
606         delete.addDeleteMarker(KeyValueUtil.ensureKeyValue(cell));
607       }
608     } else {
609       delete = new Delete(row, timestamp);
610       for (ColumnValue column: proto.getColumnValueList()) {
611         byte[] family = column.getFamily().toByteArray();
612         for (QualifierValue qv: column.getQualifierValueList()) {
613           DeleteType deleteType = qv.getDeleteType();
614           byte[] qualifier = null;
615           if (qv.hasQualifier()) {
616             qualifier = qv.getQualifier().toByteArray();
617           }
618           long ts = HConstants.LATEST_TIMESTAMP;
619           if (qv.hasTimestamp()) {
620             ts = qv.getTimestamp();
621           }
622           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
623             delete.deleteColumn(family, qualifier, ts);
624           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
625             delete.deleteColumns(family, qualifier, ts);
626           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
627             delete.deleteFamilyVersion(family, ts);
628           } else {
629             delete.deleteFamily(family, ts);
630           }
631         }
632       }
633     }
634     delete.setDurability(toDurability(proto.getDurability()));
635     for (NameBytesPair attribute: proto.getAttributeList()) {
636       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
637     }
638     return delete;
639   }
640 
641   /**
642    * Convert a protocol buffer Mutate to an Append
643    * @param cellScanner
644    * @param proto the protocol buffer Mutate to convert
645    * @return the converted client Append
646    * @throws IOException
647    */
648   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
649   throws IOException {
650     MutationType type = proto.getMutateType();
651     assert type == MutationType.APPEND : type.name();
652     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
653     Append append = null;
654     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
655     if (cellCount > 0) {
656       // The proto has metadata only and the data is separate to be found in the cellScanner.
657       if (cellScanner == null) {
658         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
659           toShortString(proto));
660       }
661       for (int i = 0; i < cellCount; i++) {
662         if (!cellScanner.advance()) {
663           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
664             " no cell returned: " + toShortString(proto));
665         }
666         Cell cell = cellScanner.current();
667         if (append == null) {
668           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
669         }
670         append.add(KeyValueUtil.ensureKeyValue(cell));
671       }
672     } else {
673       append = new Append(row);
674       for (ColumnValue column: proto.getColumnValueList()) {
675         byte[] family = column.getFamily().toByteArray();
676         for (QualifierValue qv: column.getQualifierValueList()) {
677           byte[] qualifier = qv.getQualifier().toByteArray();
678           if (!qv.hasValue()) {
679             throw new DoNotRetryIOException(
680               "Missing required field: qualifer value");
681           }
682           byte[] value = qv.getValue().toByteArray();
683           byte[] tags = null;
684           if (qv.hasTags()) {
685             tags = qv.getTags().toByteArray();
686           }
687           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
688               KeyValue.Type.Put, value, tags));
689         }
690       }
691     }
692     append.setDurability(toDurability(proto.getDurability()));
693     for (NameBytesPair attribute: proto.getAttributeList()) {
694       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
695     }
696     return append;
697   }
698 
699   /**
700    * Convert a MutateRequest to Mutation
701    *
702    * @param proto the protocol buffer Mutate to convert
703    * @return the converted Mutation
704    * @throws IOException
705    */
706   public static Mutation toMutation(final MutationProto proto) throws IOException {
707     MutationType type = proto.getMutateType();
708     if (type == MutationType.APPEND) {
709       return toAppend(proto, null);
710     }
711     if (type == MutationType.DELETE) {
712       return toDelete(proto, null);
713     }
714     if (type == MutationType.PUT) {
715       return toPut(proto, null);
716     }
717     throw new IOException("Unknown mutation type " + type);
718   }
719 
720   /**
721    * Convert a protocol buffer Mutate to an Increment
722    *
723    * @param proto the protocol buffer Mutate to convert
724    * @return the converted client Increment
725    * @throws IOException
726    */
727   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
728   throws IOException {
729     MutationType type = proto.getMutateType();
730     assert type == MutationType.INCREMENT : type.name();
731     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
732     Increment increment = null;
733     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
734     if (cellCount > 0) {
735       // The proto has metadata only and the data is separate to be found in the cellScanner.
736       if (cellScanner == null) {
737         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
738           TextFormat.shortDebugString(proto));
739       }
740       for (int i = 0; i < cellCount; i++) {
741         if (!cellScanner.advance()) {
742           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
743             " no cell returned: " + TextFormat.shortDebugString(proto));
744         }
745         Cell cell = cellScanner.current();
746         if (increment == null) {
747           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
748         }
749         increment.add(KeyValueUtil.ensureKeyValue(cell));
750       }
751     } else {
752       increment = new Increment(row);
753       for (ColumnValue column: proto.getColumnValueList()) {
754         byte[] family = column.getFamily().toByteArray();
755         for (QualifierValue qv: column.getQualifierValueList()) {
756           byte[] qualifier = qv.getQualifier().toByteArray();
757           if (!qv.hasValue()) {
758             throw new DoNotRetryIOException("Missing required field: qualifer value");
759           }
760           byte[] value = qv.getValue().toByteArray();
761           byte[] tags = null;
762           if (qv.hasTags()) {
763             tags = qv.getTags().toByteArray();
764           }
765           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
766               KeyValue.Type.Put, value, tags));
767         }
768       }
769     }
770     if (proto.hasTimeRange()) {
771       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
772       long minStamp = 0;
773       long maxStamp = Long.MAX_VALUE;
774       if (timeRange.hasFrom()) {
775         minStamp = timeRange.getFrom();
776       }
777       if (timeRange.hasTo()) {
778         maxStamp = timeRange.getTo();
779       }
780       increment.setTimeRange(minStamp, maxStamp);
781     }
782     increment.setDurability(toDurability(proto.getDurability()));
783     for (NameBytesPair attribute : proto.getAttributeList()) {
784       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
785     }
786     return increment;
787   }
788 
789   /**
790    * Convert a client Scan to a protocol buffer Scan
791    *
792    * @param scan the client Scan to convert
793    * @return the converted protocol buffer Scan
794    * @throws IOException
795    */
796   public static ClientProtos.Scan toScan(
797       final Scan scan) throws IOException {
798     ClientProtos.Scan.Builder scanBuilder =
799       ClientProtos.Scan.newBuilder();
800     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
801     if (scan.getBatch() > 0) {
802       scanBuilder.setBatchSize(scan.getBatch());
803     }
804     if (scan.getMaxResultSize() > 0) {
805       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
806     }
807     if (scan.isSmall()) {
808       scanBuilder.setSmall(scan.isSmall());
809     }
810     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
811     if (loadColumnFamiliesOnDemand != null) {
812       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
813     }
814     scanBuilder.setMaxVersions(scan.getMaxVersions());
815     TimeRange timeRange = scan.getTimeRange();
816     if (!timeRange.isAllTime()) {
817       HBaseProtos.TimeRange.Builder timeRangeBuilder =
818         HBaseProtos.TimeRange.newBuilder();
819       timeRangeBuilder.setFrom(timeRange.getMin());
820       timeRangeBuilder.setTo(timeRange.getMax());
821       scanBuilder.setTimeRange(timeRangeBuilder.build());
822     }
823     Map<String, byte[]> attributes = scan.getAttributesMap();
824     if (!attributes.isEmpty()) {
825       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
826       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
827         attributeBuilder.setName(attribute.getKey());
828         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
829         scanBuilder.addAttribute(attributeBuilder.build());
830       }
831     }
832     byte[] startRow = scan.getStartRow();
833     if (startRow != null && startRow.length > 0) {
834       scanBuilder.setStartRow(HBaseZeroCopyByteString.wrap(startRow));
835     }
836     byte[] stopRow = scan.getStopRow();
837     if (stopRow != null && stopRow.length > 0) {
838       scanBuilder.setStopRow(HBaseZeroCopyByteString.wrap(stopRow));
839     }
840     if (scan.hasFilter()) {
841       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
842     }
843     if (scan.hasFamilies()) {
844       Column.Builder columnBuilder = Column.newBuilder();
845       for (Map.Entry<byte[],NavigableSet<byte []>>
846           family: scan.getFamilyMap().entrySet()) {
847         columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
848         NavigableSet<byte []> qualifiers = family.getValue();
849         columnBuilder.clearQualifier();
850         if (qualifiers != null && qualifiers.size() > 0) {
851           for (byte [] qualifier: qualifiers) {
852             columnBuilder.addQualifier(HBaseZeroCopyByteString.wrap(qualifier));
853           }
854         }
855         scanBuilder.addColumn(columnBuilder.build());
856       }
857     }
858     if (scan.getMaxResultsPerColumnFamily() >= 0) {
859       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
860     }
861     if (scan.getRowOffsetPerColumnFamily() > 0) {
862       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
863     }
864     if (scan.isReversed()) {
865       scanBuilder.setReversed(scan.isReversed());
866     }
867     return scanBuilder.build();
868   }
869 
870   /**
871    * Convert a protocol buffer Scan to a client Scan
872    *
873    * @param proto the protocol buffer Scan to convert
874    * @return the converted client Scan
875    * @throws IOException
876    */
877   public static Scan toScan(
878       final ClientProtos.Scan proto) throws IOException {
879     byte [] startRow = HConstants.EMPTY_START_ROW;
880     byte [] stopRow  = HConstants.EMPTY_END_ROW;
881     if (proto.hasStartRow()) {
882       startRow = proto.getStartRow().toByteArray();
883     }
884     if (proto.hasStopRow()) {
885       stopRow = proto.getStopRow().toByteArray();
886     }
887     Scan scan = new Scan(startRow, stopRow);
888     if (proto.hasCacheBlocks()) {
889       scan.setCacheBlocks(proto.getCacheBlocks());
890     }
891     if (proto.hasMaxVersions()) {
892       scan.setMaxVersions(proto.getMaxVersions());
893     }
894     if (proto.hasStoreLimit()) {
895       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
896     }
897     if (proto.hasStoreOffset()) {
898       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
899     }
900     if (proto.hasLoadColumnFamiliesOnDemand()) {
901       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
902     }
903     if (proto.hasTimeRange()) {
904       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
905       long minStamp = 0;
906       long maxStamp = Long.MAX_VALUE;
907       if (timeRange.hasFrom()) {
908         minStamp = timeRange.getFrom();
909       }
910       if (timeRange.hasTo()) {
911         maxStamp = timeRange.getTo();
912       }
913       scan.setTimeRange(minStamp, maxStamp);
914     }
915     if (proto.hasFilter()) {
916       FilterProtos.Filter filter = proto.getFilter();
917       scan.setFilter(ProtobufUtil.toFilter(filter));
918     }
919     if (proto.hasBatchSize()) {
920       scan.setBatch(proto.getBatchSize());
921     }
922     if (proto.hasMaxResultSize()) {
923       scan.setMaxResultSize(proto.getMaxResultSize());
924     }
925     if (proto.hasSmall()) {
926       scan.setSmall(proto.getSmall());
927     }
928     for (NameBytesPair attribute: proto.getAttributeList()) {
929       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
930     }
931     if (proto.getColumnCount() > 0) {
932       for (Column column: proto.getColumnList()) {
933         byte[] family = column.getFamily().toByteArray();
934         if (column.getQualifierCount() > 0) {
935           for (ByteString qualifier: column.getQualifierList()) {
936             scan.addColumn(family, qualifier.toByteArray());
937           }
938         } else {
939           scan.addFamily(family);
940         }
941       }
942     }
943     if (proto.hasReversed()) {
944       scan.setReversed(proto.getReversed());
945     }
946     return scan;
947   }
948 
949   /**
950    * Create a protocol buffer Get based on a client Get.
951    *
952    * @param get the client Get
953    * @return a protocol buffer Get
954    * @throws IOException
955    */
956   public static ClientProtos.Get toGet(
957       final Get get) throws IOException {
958     ClientProtos.Get.Builder builder =
959       ClientProtos.Get.newBuilder();
960     builder.setRow(HBaseZeroCopyByteString.wrap(get.getRow()));
961     builder.setCacheBlocks(get.getCacheBlocks());
962     builder.setMaxVersions(get.getMaxVersions());
963     if (get.getFilter() != null) {
964       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
965     }
966     TimeRange timeRange = get.getTimeRange();
967     if (!timeRange.isAllTime()) {
968       HBaseProtos.TimeRange.Builder timeRangeBuilder =
969         HBaseProtos.TimeRange.newBuilder();
970       timeRangeBuilder.setFrom(timeRange.getMin());
971       timeRangeBuilder.setTo(timeRange.getMax());
972       builder.setTimeRange(timeRangeBuilder.build());
973     }
974     Map<String, byte[]> attributes = get.getAttributesMap();
975     if (!attributes.isEmpty()) {
976       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
977       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
978         attributeBuilder.setName(attribute.getKey());
979         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
980         builder.addAttribute(attributeBuilder.build());
981       }
982     }
983     if (get.hasFamilies()) {
984       Column.Builder columnBuilder = Column.newBuilder();
985       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
986       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
987         NavigableSet<byte[]> qualifiers = family.getValue();
988         columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
989         columnBuilder.clearQualifier();
990         if (qualifiers != null && qualifiers.size() > 0) {
991           for (byte[] qualifier: qualifiers) {
992             columnBuilder.addQualifier(HBaseZeroCopyByteString.wrap(qualifier));
993           }
994         }
995         builder.addColumn(columnBuilder.build());
996       }
997     }
998     if (get.getMaxResultsPerColumnFamily() >= 0) {
999       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1000     }
1001     if (get.getRowOffsetPerColumnFamily() > 0) {
1002       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1003     }
1004     if (get.isCheckExistenceOnly()){
1005       builder.setExistenceOnly(true);
1006     }
1007     if (get.isClosestRowBefore()){
1008       builder.setClosestRowBefore(true);
1009     }
1010     return builder.build();
1011   }
1012 
1013   /**
1014    * Convert a client Increment to a protobuf Mutate.
1015    *
1016    * @param increment
1017    * @return the converted mutate
1018    */
1019   public static MutationProto toMutation(
1020     final Increment increment, final MutationProto.Builder builder, long nonce) {
1021     builder.setRow(HBaseZeroCopyByteString.wrap(increment.getRow()));
1022     builder.setMutateType(MutationType.INCREMENT);
1023     builder.setDurability(toDurability(increment.getDurability()));
1024     if (nonce != HConstants.NO_NONCE) {
1025       builder.setNonce(nonce);
1026     }
1027     TimeRange timeRange = increment.getTimeRange();
1028     if (!timeRange.isAllTime()) {
1029       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1030         HBaseProtos.TimeRange.newBuilder();
1031       timeRangeBuilder.setFrom(timeRange.getMin());
1032       timeRangeBuilder.setTo(timeRange.getMax());
1033       builder.setTimeRange(timeRangeBuilder.build());
1034     }
1035     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1036     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1037     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1038       columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
1039       columnBuilder.clearQualifierValue();
1040       List<Cell> values = family.getValue();
1041       if (values != null && values.size() > 0) {
1042         for (Cell cell: values) {
1043           KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1044           valueBuilder.setQualifier(HBaseZeroCopyByteString.wrap(
1045               kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1046           valueBuilder.setValue(HBaseZeroCopyByteString.wrap(
1047               kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1048           if (kv.getTagsLength() > 0) {
1049             valueBuilder.setTags(HBaseZeroCopyByteString.wrap(kv.getTagsArray(),
1050                 kv.getTagsOffset(), kv.getTagsLength()));
1051           }
1052           columnBuilder.addQualifierValue(valueBuilder.build());
1053         }
1054       }
1055       builder.addColumnValue(columnBuilder.build());
1056     }
1057     Map<String, byte[]> attributes = increment.getAttributesMap();
1058     if (!attributes.isEmpty()) {
1059       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1060       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1061         attributeBuilder.setName(attribute.getKey());
1062         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
1063         builder.addAttribute(attributeBuilder.build());
1064       }
1065     }
1066     return builder.build();
1067   }
1068 
1069   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1070     throws IOException {
1071     return toMutation(type, mutation, HConstants.NO_NONCE);
1072   }
1073 
1074   /**
1075    * Create a protocol buffer Mutate based on a client Mutation
1076    *
1077    * @param type
1078    * @param mutation
1079    * @return a protobuf'd Mutation
1080    * @throws IOException
1081    */
1082   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1083     final long nonce) throws IOException {
1084     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1085   }
1086 
1087   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1088       MutationProto.Builder builder) throws IOException {
1089     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1090   }
1091 
1092   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1093       MutationProto.Builder builder, long nonce)
1094   throws IOException {
1095     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1096     if (nonce != HConstants.NO_NONCE) {
1097       builder.setNonce(nonce);
1098     }
1099     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1100     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1101     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1102       columnBuilder.clear();
1103       columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
1104       for (Cell cell: family.getValue()) {
1105         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1106         valueBuilder.setQualifier(HBaseZeroCopyByteString.wrap(
1107             kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1108         valueBuilder.setValue(HBaseZeroCopyByteString.wrap(
1109             kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1110         valueBuilder.setTimestamp(kv.getTimestamp());
1111         if(cell.getTagsLength() > 0) {
1112           valueBuilder.setTags(ByteString.copyFrom(CellUtil.getTagArray(kv)));
1113         }
1114         if (type == MutationType.DELETE) {
1115           KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
1116           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1117         }
1118         columnBuilder.addQualifierValue(valueBuilder.build());
1119       }
1120       builder.addColumnValue(columnBuilder.build());
1121     }
1122     return builder.build();
1123   }
1124 
1125   /**
1126    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1127    * Understanding is that the Cell will be transported other than via protobuf.
1128    * @param type
1129    * @param mutation
1130    * @param builder
1131    * @return a protobuf'd Mutation
1132    * @throws IOException
1133    */
1134   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1135       final MutationProto.Builder builder)  throws IOException {
1136     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1137   }
1138 
1139   /**
1140    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1141    * Understanding is that the Cell will be transported other than via protobuf.
1142    * @param type
1143    * @param mutation
1144    * @return a protobuf'd Mutation
1145    * @throws IOException
1146    */
1147   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1148   throws IOException {
1149     MutationProto.Builder builder =  MutationProto.newBuilder();
1150     return toMutationNoData(type, mutation, builder);
1151   }
1152 
1153   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1154       final MutationProto.Builder builder, long nonce) throws IOException {
1155     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1156     builder.setAssociatedCellCount(mutation.size());
1157     if (nonce != HConstants.NO_NONCE) {
1158       builder.setNonce(nonce);
1159     }
1160     return builder.build();
1161   }
1162 
1163   /**
1164    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1165    * {@link #toMutationNoData(MutationType, Mutation)}
1166    * @param type
1167    * @param mutation
1168    * @return A partly-filled out protobuf'd Mutation.
1169    */
1170   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1171       final Mutation mutation, MutationProto.Builder builder) {
1172     builder.setRow(HBaseZeroCopyByteString.wrap(mutation.getRow()));
1173     builder.setMutateType(type);
1174     builder.setDurability(toDurability(mutation.getDurability()));
1175     builder.setTimestamp(mutation.getTimeStamp());
1176     Map<String, byte[]> attributes = mutation.getAttributesMap();
1177     if (!attributes.isEmpty()) {
1178       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1179       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1180         attributeBuilder.setName(attribute.getKey());
1181         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
1182         builder.addAttribute(attributeBuilder.build());
1183       }
1184     }
1185     return builder;
1186   }
1187 
1188   /**
1189    * Convert a client Result to a protocol buffer Result
1190    *
1191    * @param result the client Result to convert
1192    * @return the converted protocol buffer Result
1193    */
1194   public static ClientProtos.Result toResult(final Result result) {
1195     if (result.getExists() != null) {
1196       return toResult(result.getExists());
1197     }
1198 
1199     Cell[] cells = result.rawCells();
1200     if (cells == null || cells.length == 0) {
1201       return EMPTY_RESULT_PB;
1202     }
1203 
1204     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1205     for (Cell c : cells) {
1206       builder.addCell(toCell(c));
1207     }
1208 
1209     return builder.build();
1210   }
1211 
1212   /**
1213    * Convert a client Result to a protocol buffer Result
1214    *
1215    * @param existence the client existence to send
1216    * @return the converted protocol buffer Result
1217    */
1218   public static ClientProtos.Result toResult(final boolean existence) {
1219     return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1220   }
1221 
1222   /**
1223    * Convert a client Result to a protocol buffer Result.
1224    * The pb Result does not include the Cell data.  That is for transport otherwise.
1225    *
1226    * @param result the client Result to convert
1227    * @return the converted protocol buffer Result
1228    */
1229   public static ClientProtos.Result toResultNoData(final Result result) {
1230     if (result.getExists() != null) return toResult(result.getExists());
1231     int size = result.size();
1232     if (size == 0) return EMPTY_RESULT_PB;
1233     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1234     builder.setAssociatedCellCount(size);
1235     return builder.build();
1236   }
1237 
1238   /**
1239    * Convert a protocol buffer Result to a client Result
1240    *
1241    * @param proto the protocol buffer Result to convert
1242    * @return the converted client Result
1243    */
1244   public static Result toResult(final ClientProtos.Result proto) {
1245     if (proto.hasExists()) {
1246       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1247     }
1248 
1249     List<CellProtos.Cell> values = proto.getCellList();
1250     if (values.isEmpty()){
1251       return EMPTY_RESULT;
1252     }
1253 
1254     List<Cell> cells = new ArrayList<Cell>(values.size());
1255     for (CellProtos.Cell c : values) {
1256       cells.add(toCell(c));
1257     }
1258     return Result.create(cells, null);
1259   }
1260 
1261   /**
1262    * Convert a protocol buffer Result to a client Result
1263    *
1264    * @param proto the protocol buffer Result to convert
1265    * @param scanner Optional cell scanner.
1266    * @return the converted client Result
1267    * @throws IOException
1268    */
1269   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1270   throws IOException {
1271     List<CellProtos.Cell> values = proto.getCellList();
1272 
1273     if (proto.hasExists()) {
1274       if ((values != null && !values.isEmpty()) ||
1275           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1276         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1277       }
1278       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1279     }
1280 
1281     // TODO: Unit test that has some Cells in scanner and some in the proto.
1282     List<Cell> cells = null;
1283     if (proto.hasAssociatedCellCount()) {
1284       int count = proto.getAssociatedCellCount();
1285       cells = new ArrayList<Cell>(count + values.size());
1286       for (int i = 0; i < count; i++) {
1287         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1288         cells.add(scanner.current());
1289       }
1290     }
1291 
1292     if (!values.isEmpty()){
1293       if (cells == null) cells = new ArrayList<Cell>(values.size());
1294       for (CellProtos.Cell c: values) {
1295         cells.add(toCell(c));
1296       }
1297     }
1298 
1299     return (cells == null || cells.isEmpty()) ? EMPTY_RESULT : Result.create(cells, null);
1300   }
1301 
1302 
1303   /**
1304    * Convert a ByteArrayComparable to a protocol buffer Comparator
1305    *
1306    * @param comparator the ByteArrayComparable to convert
1307    * @return the converted protocol buffer Comparator
1308    */
1309   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1310     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1311     builder.setName(comparator.getClass().getName());
1312     builder.setSerializedComparator(HBaseZeroCopyByteString.wrap(comparator.toByteArray()));
1313     return builder.build();
1314   }
1315 
1316   /**
1317    * Convert a protocol buffer Comparator to a ByteArrayComparable
1318    *
1319    * @param proto the protocol buffer Comparator to convert
1320    * @return the converted ByteArrayComparable
1321    */
1322   @SuppressWarnings("unchecked")
1323   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1324   throws IOException {
1325     String type = proto.getName();
1326     String funcName = "parseFrom";
1327     byte [] value = proto.getSerializedComparator().toByteArray();
1328     try {
1329       Class<? extends ByteArrayComparable> c =
1330         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1331       Method parseFrom = c.getMethod(funcName, byte[].class);
1332       if (parseFrom == null) {
1333         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1334       }
1335       return (ByteArrayComparable)parseFrom.invoke(null, value);
1336     } catch (Exception e) {
1337       throw new IOException(e);
1338     }
1339   }
1340 
1341   /**
1342    * Convert a protocol buffer Filter to a client Filter
1343    *
1344    * @param proto the protocol buffer Filter to convert
1345    * @return the converted Filter
1346    */
1347   @SuppressWarnings("unchecked")
1348   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1349     String type = proto.getName();
1350     final byte [] value = proto.getSerializedFilter().toByteArray();
1351     String funcName = "parseFrom";
1352     try {
1353       Class<? extends Filter> c =
1354         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1355       Method parseFrom = c.getMethod(funcName, byte[].class);
1356       if (parseFrom == null) {
1357         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1358       }
1359       return (Filter)parseFrom.invoke(c, value);
1360     } catch (Exception e) {
1361       throw new IOException(e);
1362     }
1363   }
1364 
1365   /**
1366    * Convert a client Filter to a protocol buffer Filter
1367    *
1368    * @param filter the Filter to convert
1369    * @return the converted protocol buffer Filter
1370    */
1371   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1372     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1373     builder.setName(filter.getClass().getName());
1374     builder.setSerializedFilter(HBaseZeroCopyByteString.wrap(filter.toByteArray()));
1375     return builder.build();
1376   }
1377 
1378   /**
1379    * Convert a delete KeyValue type to protocol buffer DeleteType.
1380    *
1381    * @param type
1382    * @return protocol buffer DeleteType
1383    * @throws IOException
1384    */
1385   public static DeleteType toDeleteType(
1386       KeyValue.Type type) throws IOException {
1387     switch (type) {
1388     case Delete:
1389       return DeleteType.DELETE_ONE_VERSION;
1390     case DeleteColumn:
1391       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1392     case DeleteFamily:
1393       return DeleteType.DELETE_FAMILY;
1394     case DeleteFamilyVersion:
1395       return DeleteType.DELETE_FAMILY_VERSION;
1396     default:
1397         throw new IOException("Unknown delete type: " + type);
1398     }
1399   }
1400 
1401   /**
1402    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1403    *
1404    * @param parameter the protocol buffer Parameter to convert
1405    * @return the converted Exception
1406    * @throws IOException if failed to deserialize the parameter
1407    */
1408   @SuppressWarnings("unchecked")
1409   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1410     if (parameter == null || !parameter.hasValue()) return null;
1411     String desc = parameter.getValue().toStringUtf8();
1412     String type = parameter.getName();
1413     try {
1414       Class<? extends Throwable> c =
1415         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1416       Constructor<? extends Throwable> cn = null;
1417       try {
1418         cn = c.getDeclaredConstructor(String.class);
1419         return cn.newInstance(desc);
1420       } catch (NoSuchMethodException e) {
1421         // Could be a raw RemoteException. See HBASE-8987.
1422         cn = c.getDeclaredConstructor(String.class, String.class);
1423         return cn.newInstance(type, desc);
1424       }
1425     } catch (Exception e) {
1426       throw new IOException(e);
1427     }
1428   }
1429 
1430 // Start helpers for Client
1431 
1432   /**
1433    * A helper to invoke a Get using client protocol. Uses the standard (i.e. not configured) {@link PayloadCarryingRpcController} semantics
1434    *
1435    * @param client
1436    * @param regionName
1437    * @param get
1438    * @return the result of the Get
1439    * @throws IOException
1440    */
1441   public static Result get(final ClientService.BlockingInterface client,
1442       final byte[] regionName, final Get get) throws IOException {
1443     return get(client, regionName, get, null);
1444   }
1445   
1446   /**
1447    * A helper to invoke a Get using client protocol.
1448    *
1449    * @param client
1450    * @param regionName
1451    * @param get
1452    * @param controller to use when writing the rpc
1453    * @return the result of the Get
1454    * @throws IOException
1455    */
1456   public static Result get(final ClientService.BlockingInterface client, final byte[] regionName,
1457       final Get get, PayloadCarryingRpcController controller) throws IOException {
1458     GetRequest request =
1459       RequestConverter.buildGetRequest(regionName, get);
1460     try {
1461       GetResponse response = client.get(controller, request);
1462       if (response == null) return null;
1463       return toResult(response.getResult());
1464     } catch (ServiceException se) {
1465       throw getRemoteException(se);
1466     }
1467   }
1468 
1469   /**
1470    * A helper to get a row of the closet one before using client protocol without setting any
1471    * special (i.e. configured) {@link PayloadCarryingRpcController}
1472    * @param client
1473    * @param regionName
1474    * @param row
1475    * @param family
1476    * @return the row or the closestRowBefore if it doesn't exist
1477    * @throws IOException
1478    */
1479   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1480       final byte[] regionName, final byte[] row, final byte[] family) throws IOException {
1481     return getRowOrBefore(client, regionName, row, family, null);
1482   }
1483   
1484   /**
1485    * A helper to get a row of the closet one before using client protocol.
1486    * @param client
1487    * @param regionName
1488    * @param row
1489    * @param family
1490    * @param payloadCarryingRpcController
1491    * @return the row or the closestRowBefore if it doesn't exist
1492    * @throws IOException
1493    */
1494   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1495       final byte[] regionName, final byte[] row, final byte[] family,
1496       PayloadCarryingRpcController payloadCarryingRpcController) throws IOException {
1497     GetRequest request =
1498       RequestConverter.buildGetRowOrBeforeRequest(
1499         regionName, row, family);
1500     try {
1501       GetResponse response = client.get(payloadCarryingRpcController, request);
1502       if (!response.hasResult()) return null;
1503       return toResult(response.getResult());
1504     } catch (ServiceException se) {
1505       throw getRemoteException(se);
1506     }
1507   }
1508 
1509   /**
1510    * A helper to bulk load a list of HFiles using client protocol.
1511    *
1512    * @param client
1513    * @param familyPaths
1514    * @param regionName
1515    * @param assignSeqNum
1516    * @return true if all are loaded
1517    * @throws IOException
1518    */
1519   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1520       final List<Pair<byte[], String>> familyPaths,
1521       final byte[] regionName, boolean assignSeqNum) throws IOException {
1522     return bulkLoadHFile(client, familyPaths, regionName, assignSeqNum, null);
1523   }
1524     
1525   /**
1526    * A helper to bulk load a list of HFiles using client protocol.
1527    *
1528    * @param client
1529    * @param familyPaths
1530    * @param regionName
1531    * @param assignSeqNum
1532    * @param controller
1533    * @return true if all are loaded
1534    * @throws IOException
1535    */
1536   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1537       final List<Pair<byte[], String>> familyPaths, final byte[] regionName, boolean assignSeqNum,
1538       PayloadCarryingRpcController controller) throws IOException {
1539     BulkLoadHFileRequest request =
1540       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1541     try {
1542       BulkLoadHFileResponse response =
1543         client.bulkLoadHFile(controller, request);
1544       return response.getLoaded();
1545     } catch (ServiceException se) {
1546       throw getRemoteException(se);
1547     }
1548   }
1549 
1550   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1551       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1552     return execService(client, call, regionName, null);
1553   }
1554 
1555   public static CoprocessorServiceResponse execService(
1556       final ClientService.BlockingInterface client, final CoprocessorServiceCall call,
1557       final byte[] regionName, PayloadCarryingRpcController controller) throws IOException {
1558     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1559         .setCall(call).setRegion(
1560             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1561     try {
1562       CoprocessorServiceResponse response =
1563           client.execService(controller, request);
1564       return response;
1565     } catch (ServiceException se) {
1566       throw getRemoteException(se);
1567     }
1568   }
1569 
1570   public static CoprocessorServiceResponse execService(
1571     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1572   throws IOException {
1573     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1574         .setCall(call).setRegion(
1575             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1576     try {
1577       CoprocessorServiceResponse response =
1578           client.execMasterService(null, request);
1579       return response;
1580     } catch (ServiceException se) {
1581       throw getRemoteException(se);
1582     }
1583   }
1584 
1585   @SuppressWarnings("unchecked")
1586   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1587       throws Exception {
1588     return (T)Methods.call(service, null, "newStub",
1589         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1590   }
1591 
1592 // End helpers for Client
1593 // Start helpers for Admin
1594 
1595   /**
1596    * A helper to retrieve region info given a region name
1597    * using admin protocol.
1598    *
1599    * @param admin
1600    * @param regionName
1601    * @return the retrieved region info
1602    * @throws IOException
1603    */
1604   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1605       final byte[] regionName) throws IOException {
1606     try {
1607       GetRegionInfoRequest request =
1608         RequestConverter.buildGetRegionInfoRequest(regionName);
1609       GetRegionInfoResponse response =
1610         admin.getRegionInfo(null, request);
1611       return HRegionInfo.convert(response.getRegionInfo());
1612     } catch (ServiceException se) {
1613       throw getRemoteException(se);
1614     }
1615   }
1616 
1617   /**
1618    * A helper to close a region given a region name
1619    * using admin protocol.
1620    *
1621    * @param admin
1622    * @param regionName
1623    * @param transitionInZK
1624    * @throws IOException
1625    */
1626   public static void closeRegion(final AdminService.BlockingInterface admin,
1627       final ServerName server, final byte[] regionName, final boolean transitionInZK) throws IOException {
1628     CloseRegionRequest closeRegionRequest =
1629       RequestConverter.buildCloseRegionRequest(server, regionName, transitionInZK);
1630     try {
1631       admin.closeRegion(null, closeRegionRequest);
1632     } catch (ServiceException se) {
1633       throw getRemoteException(se);
1634     }
1635   }
1636 
1637   /**
1638    * A helper to close a region given a region name
1639    * using admin protocol.
1640    *
1641    * @param admin
1642    * @param regionName
1643    * @param versionOfClosingNode
1644    * @return true if the region is closed
1645    * @throws IOException
1646    */
1647   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1648       final ServerName server,
1649       final byte[] regionName,
1650       final int versionOfClosingNode, final ServerName destinationServer,
1651       final boolean transitionInZK) throws IOException {
1652     CloseRegionRequest closeRegionRequest =
1653       RequestConverter.buildCloseRegionRequest(server,
1654         regionName, versionOfClosingNode, destinationServer, transitionInZK);
1655     try {
1656       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1657       return ResponseConverter.isClosed(response);
1658     } catch (ServiceException se) {
1659       throw getRemoteException(se);
1660     }
1661   }
1662 
1663 
1664   /**
1665    * A helper to open a region using admin protocol.
1666    * @param admin
1667    * @param region
1668    * @throws IOException
1669    */
1670   public static void openRegion(final AdminService.BlockingInterface admin,
1671       ServerName server, final HRegionInfo region) throws IOException {
1672     OpenRegionRequest request =
1673       RequestConverter.buildOpenRegionRequest(server, region, -1, null);
1674     try {
1675       admin.openRegion(null, request);
1676     } catch (ServiceException se) {
1677       throw ProtobufUtil.getRemoteException(se);
1678     }
1679   }
1680 
1681   /**
1682    * A helper to get the all the online regions on a region
1683    * server using admin protocol.
1684    *
1685    * @param admin
1686    * @return a list of online region info
1687    * @throws IOException
1688    */
1689   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1690   throws IOException {
1691     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1692     GetOnlineRegionResponse response = null;
1693     try {
1694       response = admin.getOnlineRegion(null, request);
1695     } catch (ServiceException se) {
1696       throw getRemoteException(se);
1697     }
1698     return getRegionInfos(response);
1699   }
1700 
1701   /**
1702    * Get the list of region info from a GetOnlineRegionResponse
1703    *
1704    * @param proto the GetOnlineRegionResponse
1705    * @return the list of region info or null if <code>proto</code> is null
1706    */
1707   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1708     if (proto == null) return null;
1709     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1710     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1711       regionInfos.add(HRegionInfo.convert(regionInfo));
1712     }
1713     return regionInfos;
1714   }
1715 
1716   /**
1717    * A helper to get the info of a region server using admin protocol.
1718    *
1719    * @param admin
1720    * @return the server name
1721    * @throws IOException
1722    */
1723   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1724   throws IOException {
1725     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1726     try {
1727       GetServerInfoResponse response = admin.getServerInfo(null, request);
1728       return response.getServerInfo();
1729     } catch (ServiceException se) {
1730       throw getRemoteException(se);
1731     }
1732   }
1733 
1734   /**
1735    * A helper to get the list of files of a column family
1736    * on a given region using admin protocol.
1737    *
1738    * @param admin
1739    * @param regionName
1740    * @param family
1741    * @return the list of store files
1742    * @throws IOException
1743    */
1744   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1745       final byte[] regionName, final byte[] family)
1746   throws IOException {
1747     GetStoreFileRequest request =
1748       RequestConverter.buildGetStoreFileRequest(regionName, family);
1749     try {
1750       GetStoreFileResponse response = admin.getStoreFile(null, request);
1751       return response.getStoreFileList();
1752     } catch (ServiceException se) {
1753       throw ProtobufUtil.getRemoteException(se);
1754     }
1755   }
1756 
1757   /**
1758    * A helper to split a region using admin protocol.
1759    *
1760    * @param admin
1761    * @param hri
1762    * @param splitPoint
1763    * @throws IOException
1764    */
1765   public static void split(final AdminService.BlockingInterface admin,
1766       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1767     SplitRegionRequest request =
1768       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1769     try {
1770       admin.splitRegion(null, request);
1771     } catch (ServiceException se) {
1772       throw ProtobufUtil.getRemoteException(se);
1773     }
1774   }
1775 
1776   /**
1777    * A helper to merge regions using admin protocol. Send request to
1778    * regionserver.
1779    * @param admin
1780    * @param region_a
1781    * @param region_b
1782    * @param forcible true if do a compulsory merge, otherwise we will only merge
1783    *          two adjacent regions
1784    * @throws IOException
1785    */
1786   public static void mergeRegions(final AdminService.BlockingInterface admin,
1787       final HRegionInfo region_a, final HRegionInfo region_b,
1788       final boolean forcible) throws IOException {
1789     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1790         region_a.getRegionName(), region_b.getRegionName(),forcible);
1791     try {
1792       admin.mergeRegions(null, request);
1793     } catch (ServiceException se) {
1794       throw ProtobufUtil.getRemoteException(se);
1795     }
1796   }
1797 
1798 // End helpers for Admin
1799 
1800   /*
1801    * Get the total (read + write) requests from a RegionLoad pb
1802    * @param rl - RegionLoad pb
1803    * @return total (read + write) requests
1804    */
1805   public static long getTotalRequestsCount(RegionLoad rl) {
1806     if (rl == null) {
1807       return 0;
1808     }
1809 
1810     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1811   }
1812 
1813 
1814   /**
1815    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1816    */
1817   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1818     // Allocate arbitrary big size so we avoid resizing.
1819     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1820     baos.write(PB_MAGIC);
1821     m.writeDelimitedTo(baos);
1822     return baos.toByteArray();
1823   }
1824 
1825   /**
1826    * Converts a Permission proto to a client Permission object.
1827    *
1828    * @param proto the protobuf Permission
1829    * @return the converted Permission
1830    */
1831   public static Permission toPermission(AccessControlProtos.Permission proto) {
1832     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1833       return toTablePermission(proto);
1834     } else {
1835       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1836       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1837     }
1838   }
1839 
1840   /**
1841    * Converts a Permission proto to a client TablePermission object.
1842    *
1843    * @param proto the protobuf Permission
1844    * @return the converted TablePermission
1845    */
1846   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1847     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1848       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1849       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1850 
1851       return new TablePermission(null, null, null,
1852           actions.toArray(new Permission.Action[actions.size()]));
1853     }
1854     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1855       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1856       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1857 
1858       if(!proto.hasNamespacePermission()) {
1859         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1860       }
1861       String namespace = perm.getNamespaceName().toStringUtf8();
1862       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1863     }
1864     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1865       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1866       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1867 
1868       byte[] qualifier = null;
1869       byte[] family = null;
1870       TableName table = null;
1871 
1872       if (!perm.hasTableName()) {
1873         throw new IllegalStateException("TableName cannot be empty");
1874       }
1875       table = ProtobufUtil.toTableName(perm.getTableName());
1876 
1877       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1878       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1879 
1880       return new TablePermission(table, family, qualifier,
1881           actions.toArray(new Permission.Action[actions.size()]));
1882     }
1883     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1884   }
1885 
1886   /**
1887    * Convert a client Permission to a Permission proto
1888    *
1889    * @param perm the client Permission
1890    * @return the protobuf Permission
1891    */
1892   public static AccessControlProtos.Permission toPermission(Permission perm) {
1893     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1894     if (perm instanceof TablePermission) {
1895       TablePermission tablePerm = (TablePermission)perm;
1896       if(tablePerm.hasNamespace()) {
1897         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1898 
1899         AccessControlProtos.NamespacePermission.Builder builder =
1900             AccessControlProtos.NamespacePermission.newBuilder();
1901         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1902         Permission.Action actions[] = perm.getActions();
1903         if (actions != null) {
1904           for (Permission.Action a : actions) {
1905             builder.addAction(toPermissionAction(a));
1906           }
1907 	}
1908         ret.setNamespacePermission(builder);
1909         return ret.build();
1910       } else if (tablePerm.hasTable()) {
1911         ret.setType(AccessControlProtos.Permission.Type.Table);
1912 
1913         AccessControlProtos.TablePermission.Builder builder =
1914             AccessControlProtos.TablePermission.newBuilder();
1915         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
1916         if (tablePerm.hasFamily()) {
1917           builder.setFamily(HBaseZeroCopyByteString.wrap(tablePerm.getFamily()));
1918         }
1919         if (tablePerm.hasQualifier()) {
1920           builder.setQualifier(HBaseZeroCopyByteString.wrap(tablePerm.getQualifier()));
1921         }
1922         Permission.Action actions[] = perm.getActions();
1923         if (actions != null) {
1924           for (Permission.Action a : actions) {
1925             builder.addAction(toPermissionAction(a));
1926           }
1927         }
1928         ret.setTablePermission(builder);
1929         return ret.build();
1930       }
1931     }
1932 
1933     ret.setType(AccessControlProtos.Permission.Type.Global);
1934 
1935     AccessControlProtos.GlobalPermission.Builder builder =
1936         AccessControlProtos.GlobalPermission.newBuilder();
1937     Permission.Action actions[] = perm.getActions();
1938     if (actions != null) {
1939       for (Permission.Action a: actions) {
1940         builder.addAction(toPermissionAction(a));
1941       }
1942     }
1943     ret.setGlobalPermission(builder);
1944     return ret.build();
1945   }
1946 
1947   /**
1948    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
1949    *
1950    * @param protoActions the list of protobuf Actions
1951    * @return the converted list of Actions
1952    */
1953   public static List<Permission.Action> toPermissionActions(
1954       List<AccessControlProtos.Permission.Action> protoActions) {
1955     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
1956     for (AccessControlProtos.Permission.Action a : protoActions) {
1957       actions.add(toPermissionAction(a));
1958     }
1959     return actions;
1960   }
1961 
1962   /**
1963    * Converts a Permission.Action proto to a client Permission.Action object.
1964    *
1965    * @param action the protobuf Action
1966    * @return the converted Action
1967    */
1968   public static Permission.Action toPermissionAction(
1969       AccessControlProtos.Permission.Action action) {
1970     switch (action) {
1971       case READ:
1972         return Permission.Action.READ;
1973       case WRITE:
1974         return Permission.Action.WRITE;
1975       case EXEC:
1976         return Permission.Action.EXEC;
1977       case CREATE:
1978         return Permission.Action.CREATE;
1979       case ADMIN:
1980         return Permission.Action.ADMIN;
1981     }
1982     throw new IllegalArgumentException("Unknown action value "+action.name());
1983   }
1984 
1985   /**
1986    * Convert a client Permission.Action to a Permission.Action proto
1987    *
1988    * @param action the client Action
1989    * @return the protobuf Action
1990    */
1991   public static AccessControlProtos.Permission.Action toPermissionAction(
1992       Permission.Action action) {
1993     switch (action) {
1994       case READ:
1995         return AccessControlProtos.Permission.Action.READ;
1996       case WRITE:
1997         return AccessControlProtos.Permission.Action.WRITE;
1998       case EXEC:
1999         return AccessControlProtos.Permission.Action.EXEC;
2000       case CREATE:
2001         return AccessControlProtos.Permission.Action.CREATE;
2002       case ADMIN:
2003         return AccessControlProtos.Permission.Action.ADMIN;
2004     }
2005     throw new IllegalArgumentException("Unknown action value "+action.name());
2006   }
2007 
2008   /**
2009    * Convert a client user permission to a user permission proto
2010    *
2011    * @param perm the client UserPermission
2012    * @return the protobuf UserPermission
2013    */
2014   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2015     return AccessControlProtos.UserPermission.newBuilder()
2016         .setUser(HBaseZeroCopyByteString.wrap(perm.getUser()))
2017         .setPermission(toPermission(perm))
2018         .build();
2019   }
2020 
2021   /**
2022    * Converts a user permission proto to a client user permission object.
2023    *
2024    * @param proto the protobuf UserPermission
2025    * @return the converted UserPermission
2026    */
2027   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2028     return new UserPermission(proto.getUser().toByteArray(),
2029         toTablePermission(proto.getPermission()));
2030   }
2031 
2032   /**
2033    * Convert a ListMultimap<String, TablePermission> where key is username
2034    * to a protobuf UserPermission
2035    *
2036    * @param perm the list of user and table permissions
2037    * @return the protobuf UserTablePermissions
2038    */
2039   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2040       ListMultimap<String, TablePermission> perm) {
2041     AccessControlProtos.UsersAndPermissions.Builder builder =
2042                   AccessControlProtos.UsersAndPermissions.newBuilder();
2043     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2044       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2045                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2046       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2047       for (TablePermission tablePerm: entry.getValue()) {
2048         userPermBuilder.addPermissions(toPermission(tablePerm));
2049       }
2050       builder.addUserPermissions(userPermBuilder.build());
2051     }
2052     return builder.build();
2053   }
2054 
2055   /**
2056    * A utility used to grant a user global permissions.
2057    * <p>
2058    * It's also called by the shell, in case you want to find references.
2059    *
2060    * @param protocol the AccessControlService protocol proxy
2061    * @param userShortName the short name of the user to grant permissions
2062    * @param actions the permissions to be granted
2063    * @throws ServiceException
2064    */
2065   public static void grant(AccessControlService.BlockingInterface protocol,
2066       String userShortName, Permission.Action... actions) throws ServiceException {
2067     List<AccessControlProtos.Permission.Action> permActions =
2068         Lists.newArrayListWithCapacity(actions.length);
2069     for (Permission.Action a : actions) {
2070       permActions.add(ProtobufUtil.toPermissionAction(a));
2071     }
2072     AccessControlProtos.GrantRequest request = RequestConverter.
2073       buildGrantRequest(userShortName, permActions.toArray(
2074         new AccessControlProtos.Permission.Action[actions.length]));
2075     protocol.grant(null, request);
2076   }
2077 
2078   /**
2079    * A utility used to grant a user table permissions. The permissions will
2080    * be for a table table/column family/qualifier.
2081    * <p>
2082    * It's also called by the shell, in case you want to find references.
2083    *
2084    * @param protocol the AccessControlService protocol proxy
2085    * @param userShortName the short name of the user to grant permissions
2086    * @param tableName optional table name
2087    * @param f optional column family
2088    * @param q optional qualifier
2089    * @param actions the permissions to be granted
2090    * @throws ServiceException
2091    */
2092   public static void grant(AccessControlService.BlockingInterface protocol,
2093       String userShortName, TableName tableName, byte[] f, byte[] q,
2094       Permission.Action... actions) throws ServiceException {
2095     List<AccessControlProtos.Permission.Action> permActions =
2096         Lists.newArrayListWithCapacity(actions.length);
2097     for (Permission.Action a : actions) {
2098       permActions.add(ProtobufUtil.toPermissionAction(a));
2099     }
2100     AccessControlProtos.GrantRequest request = RequestConverter.
2101       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2102         new AccessControlProtos.Permission.Action[actions.length]));
2103     protocol.grant(null, request);
2104   }
2105 
2106   /**
2107    * A utility used to grant a user namespace permissions.
2108    * <p>
2109    * It's also called by the shell, in case you want to find references.
2110    *
2111    * @param protocol the AccessControlService protocol proxy
2112    * @param namespace the short name of the user to grant permissions
2113    * @param actions the permissions to be granted
2114    * @throws ServiceException
2115    */
2116   public static void grant(AccessControlService.BlockingInterface protocol,
2117       String userShortName, String namespace,
2118       Permission.Action... actions) throws ServiceException {
2119     List<AccessControlProtos.Permission.Action> permActions =
2120         Lists.newArrayListWithCapacity(actions.length);
2121     for (Permission.Action a : actions) {
2122       permActions.add(ProtobufUtil.toPermissionAction(a));
2123     }
2124     AccessControlProtos.GrantRequest request = RequestConverter.
2125       buildGrantRequest(userShortName, namespace, permActions.toArray(
2126         new AccessControlProtos.Permission.Action[actions.length]));
2127     protocol.grant(null, request);
2128   }
2129 
2130   /**
2131    * A utility used to revoke a user's global permissions.
2132    * <p>
2133    * It's also called by the shell, in case you want to find references.
2134    *
2135    * @param protocol the AccessControlService protocol proxy
2136    * @param userShortName the short name of the user to revoke permissions
2137    * @param actions the permissions to be revoked
2138    * @throws ServiceException
2139    */
2140   public static void revoke(AccessControlService.BlockingInterface protocol,
2141       String userShortName, Permission.Action... actions) throws ServiceException {
2142     List<AccessControlProtos.Permission.Action> permActions =
2143         Lists.newArrayListWithCapacity(actions.length);
2144     for (Permission.Action a : actions) {
2145       permActions.add(ProtobufUtil.toPermissionAction(a));
2146     }
2147     AccessControlProtos.RevokeRequest request = RequestConverter.
2148       buildRevokeRequest(userShortName, permActions.toArray(
2149         new AccessControlProtos.Permission.Action[actions.length]));
2150     protocol.revoke(null, request);
2151   }
2152 
2153   /**
2154    * A utility used to revoke a user's table permissions. The permissions will
2155    * be for a table/column family/qualifier.
2156    * <p>
2157    * It's also called by the shell, in case you want to find references.
2158    *
2159    * @param protocol the AccessControlService protocol proxy
2160    * @param userShortName the short name of the user to revoke permissions
2161    * @param tableName optional table name
2162    * @param f optional column family
2163    * @param q optional qualifier
2164    * @param actions the permissions to be revoked
2165    * @throws ServiceException
2166    */
2167   public static void revoke(AccessControlService.BlockingInterface protocol,
2168       String userShortName, TableName tableName, byte[] f, byte[] q,
2169       Permission.Action... actions) throws ServiceException {
2170     List<AccessControlProtos.Permission.Action> permActions =
2171         Lists.newArrayListWithCapacity(actions.length);
2172     for (Permission.Action a : actions) {
2173       permActions.add(ProtobufUtil.toPermissionAction(a));
2174     }
2175     AccessControlProtos.RevokeRequest request = RequestConverter.
2176       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2177         new AccessControlProtos.Permission.Action[actions.length]));
2178     protocol.revoke(null, request);
2179   }
2180 
2181   /**
2182    * A utility used to revoke a user's namespace permissions.
2183    * <p>
2184    * It's also called by the shell, in case you want to find references.
2185    *
2186    * @param protocol the AccessControlService protocol proxy
2187    * @param userShortName the short name of the user to revoke permissions
2188    * @param namespace optional table name
2189    * @param actions the permissions to be revoked
2190    * @throws ServiceException
2191    */
2192   public static void revoke(AccessControlService.BlockingInterface protocol,
2193       String userShortName, String namespace,
2194       Permission.Action... actions) throws ServiceException {
2195     List<AccessControlProtos.Permission.Action> permActions =
2196         Lists.newArrayListWithCapacity(actions.length);
2197     for (Permission.Action a : actions) {
2198       permActions.add(ProtobufUtil.toPermissionAction(a));
2199     }
2200     AccessControlProtos.RevokeRequest request = RequestConverter.
2201       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2202         new AccessControlProtos.Permission.Action[actions.length]));
2203     protocol.revoke(null, request);
2204   }
2205 
2206   /**
2207    * A utility used to get user's global permissions.
2208    * <p>
2209    * It's also called by the shell, in case you want to find references.
2210    *
2211    * @param protocol the AccessControlService protocol proxy
2212    * @throws ServiceException
2213    */
2214   public static List<UserPermission> getUserPermissions(
2215       AccessControlService.BlockingInterface protocol) throws ServiceException {
2216     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2217       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2218     builder.setType(AccessControlProtos.Permission.Type.Global);
2219     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2220     AccessControlProtos.GetUserPermissionsResponse response =
2221       protocol.getUserPermissions(null, request);
2222     List<UserPermission> perms = new ArrayList<UserPermission>();
2223     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2224       perms.add(ProtobufUtil.toUserPermission(perm));
2225     }
2226     return perms;
2227   }
2228 
2229   /**
2230    * A utility used to get user table permissions.
2231    * <p>
2232    * It's also called by the shell, in case you want to find references.
2233    *
2234    * @param protocol the AccessControlService protocol proxy
2235    * @param t optional table name
2236    * @throws ServiceException
2237    */
2238   public static List<UserPermission> getUserPermissions(
2239       AccessControlService.BlockingInterface protocol,
2240       TableName t) throws ServiceException {
2241     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2242       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2243     if (t != null) {
2244       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2245     }
2246     builder.setType(AccessControlProtos.Permission.Type.Table);
2247     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2248     AccessControlProtos.GetUserPermissionsResponse response =
2249       protocol.getUserPermissions(null, request);
2250     List<UserPermission> perms = new ArrayList<UserPermission>();
2251     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2252       perms.add(ProtobufUtil.toUserPermission(perm));
2253     }
2254     return perms;
2255   }
2256 
2257   /**
2258    * A utility used to get permissions for selected namespace.
2259    * <p>
2260    * It's also called by the shell, in case you want to find references.
2261    *
2262    * @param protocol the AccessControlService protocol proxy
2263    * @param namespace name of the namespace
2264    * @throws ServiceException
2265    */
2266   public static List<UserPermission> getUserPermissions(
2267       AccessControlService.BlockingInterface protocol,
2268       byte[] namespace) throws ServiceException {
2269     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2270       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2271     if (namespace != null) {
2272       builder.setNamespaceName(HBaseZeroCopyByteString.wrap(namespace));
2273     }
2274     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2275     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2276     AccessControlProtos.GetUserPermissionsResponse response =
2277       protocol.getUserPermissions(null, request);
2278     List<UserPermission> perms = new ArrayList<UserPermission>();
2279     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2280       perms.add(ProtobufUtil.toUserPermission(perm));
2281     }
2282     return perms;
2283   }
2284 
2285   /**
2286    * Convert a protobuf UserTablePermissions to a
2287    * ListMultimap<String, TablePermission> where key is username.
2288    *
2289    * @param proto the protobuf UserPermission
2290    * @return the converted UserPermission
2291    */
2292   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2293       AccessControlProtos.UsersAndPermissions proto) {
2294     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2295     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2296 
2297     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2298       userPerm = proto.getUserPermissions(i);
2299       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2300         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2301         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2302       }
2303     }
2304 
2305     return perms;
2306   }
2307 
2308   /**
2309    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2310    *
2311    * @param token the Token instance to copy
2312    * @return the protobuf Token message
2313    */
2314   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2315     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2316     builder.setIdentifier(HBaseZeroCopyByteString.wrap(token.getIdentifier()));
2317     builder.setPassword(HBaseZeroCopyByteString.wrap(token.getPassword()));
2318     if (token.getService() != null) {
2319       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2320     }
2321     return builder.build();
2322   }
2323 
2324   /**
2325    * Converts a protobuf Token message back into a Token instance.
2326    *
2327    * @param proto the protobuf Token message
2328    * @return the Token instance
2329    */
2330   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2331     return new Token<AuthenticationTokenIdentifier>(
2332         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2333         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2334         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2335         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2336   }
2337 
2338   /**
2339    * Find the HRegion encoded name based on a region specifier
2340    *
2341    * @param regionSpecifier the region specifier
2342    * @return the corresponding region's encoded name
2343    * @throws DoNotRetryIOException if the specifier type is unsupported
2344    */
2345   public static String getRegionEncodedName(
2346       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2347     byte[] value = regionSpecifier.getValue().toByteArray();
2348     RegionSpecifierType type = regionSpecifier.getType();
2349     switch (type) {
2350       case REGION_NAME:
2351         return HRegionInfo.encodeRegionName(value);
2352       case ENCODED_REGION_NAME:
2353         return Bytes.toString(value);
2354       default:
2355         throw new DoNotRetryIOException(
2356           "Unsupported region specifier type: " + type);
2357     }
2358   }
2359 
2360   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2361     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2362     MapReduceProtos.ScanMetrics pScanMetrics = null;
2363     try {
2364       pScanMetrics = parser.parseFrom(bytes);
2365     } catch (InvalidProtocolBufferException e) {
2366       //Ignored there are just no key values to add.
2367     }
2368     ScanMetrics scanMetrics = new ScanMetrics();
2369     if (pScanMetrics != null) {
2370       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2371         if (pair.hasName() && pair.hasValue()) {
2372           scanMetrics.setCounter(pair.getName(), pair.getValue());
2373         }
2374       }
2375     }
2376     return scanMetrics;
2377   }
2378 
2379   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2380     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2381     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2382     for (Entry<String, Long> e : metrics.entrySet()) {
2383       HBaseProtos.NameInt64Pair nameInt64Pair =
2384           HBaseProtos.NameInt64Pair.newBuilder()
2385               .setName(e.getKey())
2386               .setValue(e.getValue())
2387               .build();
2388       builder.addMetrics(nameInt64Pair);
2389     }
2390     return builder.build();
2391   }
2392 
2393   /**
2394    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2395    * This method will <strong>always</strong> throw an exception.
2396    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2397    */
2398   public static void toIOException(ServiceException se) throws IOException {
2399     if (se == null) {
2400       throw new NullPointerException("Null service exception passed!");
2401     }
2402 
2403     Throwable cause = se.getCause();
2404     if (cause != null && cause instanceof IOException) {
2405       throw (IOException)cause;
2406     }
2407     throw new IOException(se);
2408   }
2409 
2410   public static CellProtos.Cell toCell(final Cell kv) {
2411     // Doing this is going to kill us if we do it for all data passed.
2412     // St.Ack 20121205
2413     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2414     kvbuilder.setRow(HBaseZeroCopyByteString.wrap(kv.getRowArray(), kv.getRowOffset(),
2415         kv.getRowLength()));
2416     kvbuilder.setFamily(HBaseZeroCopyByteString.wrap(kv.getFamilyArray(),
2417         kv.getFamilyOffset(), kv.getFamilyLength()));
2418     kvbuilder.setQualifier(HBaseZeroCopyByteString.wrap(kv.getQualifierArray(),
2419         kv.getQualifierOffset(), kv.getQualifierLength()));
2420     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2421     kvbuilder.setTimestamp(kv.getTimestamp());
2422     kvbuilder.setValue(HBaseZeroCopyByteString.wrap(kv.getValueArray(), kv.getValueOffset(),
2423         kv.getValueLength()));
2424     return kvbuilder.build();
2425   }
2426 
2427   public static Cell toCell(final CellProtos.Cell cell) {
2428     // Doing this is going to kill us if we do it for all data passed.
2429     // St.Ack 20121205
2430     return CellUtil.createCell(cell.getRow().toByteArray(),
2431       cell.getFamily().toByteArray(),
2432       cell.getQualifier().toByteArray(),
2433       cell.getTimestamp(),
2434       (byte)cell.getCellType().getNumber(),
2435       cell.getValue().toByteArray());
2436   }
2437 
2438   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2439     HBaseProtos.NamespaceDescriptor.Builder b =
2440         HBaseProtos.NamespaceDescriptor.newBuilder()
2441             .setName(ByteString.copyFromUtf8(ns.getName()));
2442     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2443       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2444           .setName(entry.getKey())
2445           .setValue(entry.getValue()));
2446     }
2447     return b.build();
2448   }
2449 
2450   public static NamespaceDescriptor toNamespaceDescriptor(
2451       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2452     NamespaceDescriptor.Builder b =
2453       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2454     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2455       b.addConfiguration(prop.getName(), prop.getValue());
2456     }
2457     return b.build();
2458   }
2459 
2460   /**
2461    * Get an instance of the argument type declared in a class's signature. The
2462    * argument type is assumed to be a PB Message subclass, and the instance is
2463    * created using parseFrom method on the passed ByteString.
2464    * @param runtimeClass the runtime type of the class
2465    * @param position the position of the argument in the class declaration
2466    * @param b the ByteString which should be parsed to get the instance created
2467    * @return the instance
2468    * @throws IOException
2469    */
2470   @SuppressWarnings("unchecked")
2471   public static <T extends Message>
2472   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2473       throws IOException {
2474     Type type = runtimeClass.getGenericSuperclass();
2475     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2476     Class<T> classType = (Class<T>)argType;
2477     T inst;
2478     try {
2479       Method m = classType.getMethod("parseFrom", ByteString.class);
2480       inst = (T)m.invoke(null, b);
2481       return inst;
2482     } catch (SecurityException e) {
2483       throw new IOException(e);
2484     } catch (NoSuchMethodException e) {
2485       throw new IOException(e);
2486     } catch (IllegalArgumentException e) {
2487       throw new IOException(e);
2488     } catch (InvocationTargetException e) {
2489       throw new IOException(e);
2490     } catch (IllegalAccessException e) {
2491       throw new IOException(e);
2492     }
2493   }
2494 
2495   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2496       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2497     // compaction descriptor contains relative paths.
2498     // input / output paths are relative to the store dir
2499     // store dir is relative to region dir
2500     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2501         .setTableName(HBaseZeroCopyByteString.wrap(info.getTableName()))
2502         .setEncodedRegionName(HBaseZeroCopyByteString.wrap(info.getEncodedNameAsBytes()))
2503         .setFamilyName(HBaseZeroCopyByteString.wrap(family))
2504         .setStoreHomeDir(storeDir.getName()); //make relative
2505     for (Path inputPath : inputPaths) {
2506       builder.addCompactionInput(inputPath.getName()); //relative path
2507     }
2508     for (Path outputPath : outputPaths) {
2509       builder.addCompactionOutput(outputPath.getName());
2510     }
2511     return builder.build();
2512   }
2513 
2514   /**
2515    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2516    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2517    * logs. Use judiciously.
2518    * @param m
2519    * @return toString of passed <code>m</code>
2520    */
2521   public static String getShortTextFormat(Message m) {
2522     if (m == null) return "null";
2523     if (m instanceof ScanRequest) {
2524       // This should be small and safe to output.  No data.
2525       return TextFormat.shortDebugString(m);
2526     } else if (m instanceof RegionServerReportRequest) {
2527       // Print a short message only, just the servername and the requests, not the full load.
2528       RegionServerReportRequest r = (RegionServerReportRequest)m;
2529       return "server " + TextFormat.shortDebugString(r.getServer()) +
2530         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2531     } else if (m instanceof RegionServerStartupRequest) {
2532       // Should be small enough.
2533       return TextFormat.shortDebugString(m);
2534     } else if (m instanceof MutationProto) {
2535       return toShortString((MutationProto)m);
2536     } else if (m instanceof GetRequest) {
2537       GetRequest r = (GetRequest) m;
2538       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2539           ", row=" + getStringForByteString(r.getGet().getRow());
2540     } else if (m instanceof ClientProtos.MultiRequest) {
2541       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2542       // Get first set of Actions.
2543       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2544       String row = actions.getActionCount() <= 0? "":
2545         getStringForByteString(actions.getAction(0).hasGet()?
2546           actions.getAction(0).getGet().getRow():
2547           actions.getAction(0).getMutation().getRow());
2548       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2549           ", for " + r.getRegionActionCount() +
2550           " actions and 1st row key=" + row;
2551     } else if (m instanceof ClientProtos.MutateRequest) {
2552       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2553       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2554           ", row=" + getStringForByteString(r.getMutation().getRow());
2555     }
2556     return "TODO: " + m.getClass().toString();
2557   }
2558 
2559   private static String getStringForByteString(ByteString bs) {
2560     return Bytes.toStringBinary(bs.toByteArray());
2561   }
2562 
2563   /**
2564    * Print out some subset of a MutationProto rather than all of it and its data
2565    * @param proto Protobuf to print out
2566    * @return Short String of mutation proto
2567    */
2568   static String toShortString(final MutationProto proto) {
2569     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2570         ", type=" + proto.getMutateType().toString();
2571   }
2572 
2573   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2574     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2575         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2576   }
2577 
2578   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2579     return HBaseProtos.TableName.newBuilder()
2580         .setNamespace(HBaseZeroCopyByteString.wrap(tableName.getNamespace()))
2581         .setQualifier(HBaseZeroCopyByteString.wrap(tableName.getQualifier())).build();
2582   }
2583 
2584   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2585     if (tableNamesList == null) {
2586       return new TableName[0];
2587     }
2588     TableName[] tableNames = new TableName[tableNamesList.size()];
2589     for (int i = 0; i < tableNamesList.size(); i++) {
2590       tableNames[i] = toTableName(tableNamesList.get(i));
2591     }
2592     return tableNames;
2593   }
2594 
2595   /**
2596    * Convert a protocol buffer CellVisibility to a client CellVisibility
2597    *
2598    * @param proto
2599    * @return the converted client CellVisibility
2600    */
2601   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2602     if (proto == null) return null;
2603     return new CellVisibility(proto.getExpression());
2604   }
2605 
2606   /**
2607    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2608    *
2609    * @param protoBytes
2610    * @return the converted client CellVisibility
2611    * @throws DeserializationException
2612    */
2613   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2614     if (protoBytes == null) return null;
2615     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2616     ClientProtos.CellVisibility proto = null;
2617     try {
2618       proto = builder.mergeFrom(protoBytes).build();
2619     } catch (InvalidProtocolBufferException e) {
2620       throw new DeserializationException(e);
2621     }
2622     return toCellVisibility(proto);
2623   }
2624 
2625   /**
2626    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2627    *
2628    * @param cellVisibility
2629    * @return a protocol buffer CellVisibility
2630    */
2631   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2632     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2633     builder.setExpression(cellVisibility.getExpression());
2634     return builder.build();
2635   }
2636 
2637   /**
2638    * Convert a protocol buffer Authorizations to a client Authorizations
2639    *
2640    * @param proto
2641    * @return the converted client Authorizations
2642    */
2643   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2644     if (proto == null) return null;
2645     return new Authorizations(proto.getLabelList());
2646   }
2647 
2648   /**
2649    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2650    *
2651    * @param protoBytes
2652    * @return the converted client Authorizations
2653    * @throws DeserializationException
2654    */
2655   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2656     if (protoBytes == null) return null;
2657     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2658     ClientProtos.Authorizations proto = null;
2659     try {
2660       proto = builder.mergeFrom(protoBytes).build();
2661     } catch (InvalidProtocolBufferException e) {
2662       throw new DeserializationException(e);
2663     }
2664     return toAuthorizations(proto);
2665   }
2666 
2667   /**
2668    * Create a protocol buffer Authorizations based on a client Authorizations.
2669    *
2670    * @param authorizations
2671    * @return a protocol buffer Authorizations
2672    */
2673   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2674     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2675     for (String label : authorizations.getLabels()) {
2676       builder.addLabel(label);
2677     }
2678     return builder.build();
2679   }
2680 
2681   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2682       Permission perms) {
2683     return AccessControlProtos.UsersAndPermissions.newBuilder()
2684       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2685         .setUser(ByteString.copyFromUtf8(user))
2686         .addPermissions(toPermission(perms))
2687         .build())
2688       .build();
2689   }
2690 
2691   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2692       ListMultimap<String, Permission> perms) {
2693     AccessControlProtos.UsersAndPermissions.Builder builder =
2694         AccessControlProtos.UsersAndPermissions.newBuilder();
2695     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2696       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2697         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2698       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2699       for (Permission perm: entry.getValue()) {
2700         userPermBuilder.addPermissions(toPermission(perm));
2701       }
2702       builder.addUserPermissions(userPermBuilder.build());
2703     }
2704     return builder.build();
2705   }
2706 
2707   public static ListMultimap<String, Permission> toUsersAndPermissions(
2708       AccessControlProtos.UsersAndPermissions proto) {
2709     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2710     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2711         proto.getUserPermissionsList()) {
2712       String user = userPerms.getUser().toStringUtf8();
2713       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2714         result.put(user, toPermission(perm));
2715       }
2716     }
2717     return result;
2718   }
2719 }