View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.lang.reflect.Constructor;
26  import java.lang.reflect.InvocationTargetException;
27  import java.lang.reflect.Method;
28  import java.lang.reflect.ParameterizedType;
29  import java.lang.reflect.Type;
30  import java.util.ArrayList;
31  import java.util.Collection;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Map.Entry;
36  import java.util.NavigableSet;
37  
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.CellScanner;
42  import org.apache.hadoop.hbase.CellUtil;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HConstants;
47  import org.apache.hadoop.hbase.HRegionInfo;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.KeyValueUtil;
51  import org.apache.hadoop.hbase.NamespaceDescriptor;
52  import org.apache.hadoop.hbase.ServerName;
53  import org.apache.hadoop.hbase.client.Append;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.Durability;
56  import org.apache.hadoop.hbase.client.Get;
57  import org.apache.hadoop.hbase.client.Increment;
58  import org.apache.hadoop.hbase.client.Mutation;
59  import org.apache.hadoop.hbase.client.Put;
60  import org.apache.hadoop.hbase.client.Result;
61  import org.apache.hadoop.hbase.client.Scan;
62  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
63  import org.apache.hadoop.hbase.exceptions.DeserializationException;
64  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
65  import org.apache.hadoop.hbase.filter.Filter;
66  import org.apache.hadoop.hbase.io.TimeRange;
67  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
68  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
69  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
70  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
71  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
72  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
85  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
86  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
87  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
88  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
89  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
90  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
103 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
104 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
105 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
106 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
107 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
108 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
109 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
110 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.MasterAdminService;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsResponse;
114 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
115 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
117 import org.apache.hadoop.hbase.security.access.Permission;
118 import org.apache.hadoop.hbase.security.access.TablePermission;
119 import org.apache.hadoop.hbase.security.access.UserPermission;
120 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
121 import org.apache.hadoop.hbase.util.Bytes;
122 import org.apache.hadoop.hbase.util.DynamicClassLoader;
123 import org.apache.hadoop.hbase.util.Methods;
124 import org.apache.hadoop.hbase.util.Pair;
125 import org.apache.hadoop.io.Text;
126 import org.apache.hadoop.ipc.RemoteException;
127 import org.apache.hadoop.security.token.Token;
128 
129 import com.google.common.collect.ArrayListMultimap;
130 import com.google.common.collect.ListMultimap;
131 import com.google.common.collect.Lists;
132 import com.google.protobuf.ByteString;
133 import com.google.protobuf.InvalidProtocolBufferException;
134 import com.google.protobuf.Message;
135 import com.google.protobuf.RpcChannel;
136 import com.google.protobuf.Service;
137 import com.google.protobuf.ServiceException;
138 import com.google.protobuf.TextFormat;
139 
140 /**
141  * Protobufs utility.
142  */
143 public final class ProtobufUtil {
144 
145   private ProtobufUtil() {
146   }
147 
148   /**
149    * Primitive type to class mapping.
150    */
151   private final static Map<String, Class<?>>
152     PRIMITIVES = new HashMap<String, Class<?>>();
153 
154   /**
155    * Dynamic class loader to load filter/comparators
156    */
157   private final static ClassLoader CLASS_LOADER;
158 
159   static {
160     ClassLoader parent = ProtobufUtil.class.getClassLoader();
161     Configuration conf = HBaseConfiguration.create();
162     CLASS_LOADER = new DynamicClassLoader(conf, parent);
163 
164     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
165     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
166     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
167     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
168     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
169     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
170     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
171     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
172     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
173   }
174 
175   /**
176    * Magic we put ahead of a serialized protobuf message.
177    * For example, all znode content is protobuf messages with the below magic
178    * for preamble.
179    */
180   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
181   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
182 
183   /**
184    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
185    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
186    * @param bytes Bytes to decorate
187    * @return The passed <code>bytes</codes> with magic prepended (Creates a new
188    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
189    */
190   public static byte [] prependPBMagic(final byte [] bytes) {
191     return Bytes.add(PB_MAGIC, bytes);
192   }
193 
194   /**
195    * @param bytes Bytes to check.
196    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
197    */
198   public static boolean isPBMagicPrefix(final byte [] bytes) {
199     if (bytes == null || bytes.length < PB_MAGIC.length) return false;
200     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, 0, PB_MAGIC.length) == 0;
201   }
202 
203   /**
204    * @param bytes
205    * @throws DeserializationException if we are missing the pb magic prefix
206    */
207   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
208     if (!isPBMagicPrefix(bytes)) {
209       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
210     }
211   }
212 
213   /**
214    * @return Length of {@link #PB_MAGIC}
215    */
216   public static int lengthOfPBMagic() {
217     return PB_MAGIC.length;
218   }
219 
220   /**
221    * Return the IOException thrown by the remote server wrapped in
222    * ServiceException as cause.
223    *
224    * @param se ServiceException that wraps IO exception thrown by the server
225    * @return Exception wrapped in ServiceException or
226    *   a new IOException that wraps the unexpected ServiceException.
227    */
228   public static IOException getRemoteException(ServiceException se) {
229     Throwable e = se.getCause();
230     if (e == null) {
231       return new IOException(se);
232     }
233     if (e instanceof RemoteException) {
234       e = ((RemoteException)e).unwrapRemoteException();
235     }
236     return e instanceof IOException ? (IOException) e : new IOException(se);
237   }
238 
239   /**
240    * Convert a ServerName to a protocol buffer ServerName
241    *
242    * @param serverName the ServerName to convert
243    * @return the converted protocol buffer ServerName
244    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
245    */
246   public static HBaseProtos.ServerName
247       toServerName(final ServerName serverName) {
248     if (serverName == null) return null;
249     HBaseProtos.ServerName.Builder builder =
250       HBaseProtos.ServerName.newBuilder();
251     builder.setHostName(serverName.getHostname());
252     if (serverName.getPort() >= 0) {
253       builder.setPort(serverName.getPort());
254     }
255     if (serverName.getStartcode() >= 0) {
256       builder.setStartCode(serverName.getStartcode());
257     }
258     return builder.build();
259   }
260 
261   /**
262    * Convert a protocol buffer ServerName to a ServerName
263    *
264    * @param proto the protocol buffer ServerName to convert
265    * @return the converted ServerName
266    */
267   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
268     if (proto == null) return null;
269     String hostName = proto.getHostName();
270     long startCode = -1;
271     int port = -1;
272     if (proto.hasPort()) {
273       port = proto.getPort();
274     }
275     if (proto.hasStartCode()) {
276       startCode = proto.getStartCode();
277     }
278     return new ServerName(hostName, port, startCode);
279   }
280 
281   /**
282    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
283    *
284    * @param proto the GetTableDescriptorsResponse
285    * @return HTableDescriptor[]
286    */
287   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
288     if (proto == null) return null;
289 
290     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
291     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
292       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
293     }
294     return ret;
295   }
296 
297   /**
298    * get the split keys in form "byte [][]" from a CreateTableRequest proto
299    *
300    * @param proto the CreateTableRequest
301    * @return the split keys
302    */
303   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
304     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
305     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
306       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
307     }
308     return splitKeys;
309   }
310 
311   /**
312    * Convert a protobuf Durability into a client Durability
313    */
314   public static Durability toDurability(
315       final ClientProtos.MutationProto.Durability proto) {
316     switch(proto) {
317     case USE_DEFAULT:
318       return Durability.USE_DEFAULT;
319     case SKIP_WAL:
320       return Durability.SKIP_WAL;
321     case ASYNC_WAL:
322       return Durability.ASYNC_WAL;
323     case SYNC_WAL:
324       return Durability.SYNC_WAL;
325     case FSYNC_WAL:
326       return Durability.FSYNC_WAL;
327     default:
328       return Durability.USE_DEFAULT;
329     }
330   }
331 
332   /**
333    * Convert a client Durability into a protbuf Durability
334    */
335   public static ClientProtos.MutationProto.Durability toDurability(
336       final Durability d) {
337     switch(d) {
338     case USE_DEFAULT:
339       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
340     case SKIP_WAL:
341       return ClientProtos.MutationProto.Durability.SKIP_WAL;
342     case ASYNC_WAL:
343       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
344     case SYNC_WAL:
345       return ClientProtos.MutationProto.Durability.SYNC_WAL;
346     case FSYNC_WAL:
347       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
348     default:
349       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
350     }
351   }
352 
353   /**
354    * Convert a protocol buffer Get to a client Get
355    *
356    * @param proto the protocol buffer Get to convert
357    * @return the converted client Get
358    * @throws IOException
359    */
360   public static Get toGet(
361       final ClientProtos.Get proto) throws IOException {
362     if (proto == null) return null;
363     byte[] row = proto.getRow().toByteArray();
364     Get get = new Get(row);
365     if (proto.hasCacheBlocks()) {
366       get.setCacheBlocks(proto.getCacheBlocks());
367     }
368     if (proto.hasMaxVersions()) {
369       get.setMaxVersions(proto.getMaxVersions());
370     }
371     if (proto.hasStoreLimit()) {
372       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
373     }
374     if (proto.hasStoreOffset()) {
375       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
376     }
377     if (proto.hasTimeRange()) {
378       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
379       long minStamp = 0;
380       long maxStamp = Long.MAX_VALUE;
381       if (timeRange.hasFrom()) {
382         minStamp = timeRange.getFrom();
383       }
384       if (timeRange.hasTo()) {
385         maxStamp = timeRange.getTo();
386       }
387       get.setTimeRange(minStamp, maxStamp);
388     }
389     if (proto.hasFilter()) {
390       FilterProtos.Filter filter = proto.getFilter();
391       get.setFilter(ProtobufUtil.toFilter(filter));
392     }
393     for (NameBytesPair attribute: proto.getAttributeList()) {
394       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
395     }
396     if (proto.getColumnCount() > 0) {
397       for (Column column: proto.getColumnList()) {
398         byte[] family = column.getFamily().toByteArray();
399         if (column.getQualifierCount() > 0) {
400           for (ByteString qualifier: column.getQualifierList()) {
401             get.addColumn(family, qualifier.toByteArray());
402           }
403         } else {
404           get.addFamily(family);
405         }
406       }
407     }
408     return get;
409   }
410 
411   /**
412    * Convert a protocol buffer Mutate to a Put.
413    *
414    * @param proto The protocol buffer MutationProto to convert
415    * @return A client Put.
416    * @throws IOException
417    */
418   public static Put toPut(final MutationProto proto)
419   throws IOException {
420     return toPut(proto, null);
421   }
422 
423   /**
424    * Convert a protocol buffer Mutate to a Put.
425    *
426    * @param proto The protocol buffer MutationProto to convert
427    * @param cellScanner If non-null, the Cell data that goes with this proto.
428    * @return A client Put.
429    * @throws IOException
430    */
431   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
432   throws IOException {
433     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
434     MutationType type = proto.getMutateType();
435     assert type == MutationType.PUT: type.name();
436     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
437     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
438     Put put = null;
439     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
440     if (cellCount > 0) {
441       // The proto has metadata only and the data is separate to be found in the cellScanner.
442       if (cellScanner == null) {
443         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
444             toShortString(proto));
445       }
446       for (int i = 0; i < cellCount; i++) {
447         if (!cellScanner.advance()) {
448           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
449             " no cell returned: " + toShortString(proto));
450         }
451         Cell cell = cellScanner.current();
452         if (put == null) {
453           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
454         }
455         put.add(KeyValueUtil.ensureKeyValue(cell));
456       }
457     } else {
458       put = new Put(row, timestamp);
459       // The proto has the metadata and the data itself
460       for (ColumnValue column: proto.getColumnValueList()) {
461         byte[] family = column.getFamily().toByteArray();
462         for (QualifierValue qv: column.getQualifierValueList()) {
463           byte[] qualifier = qv.getQualifier().toByteArray();
464           if (!qv.hasValue()) {
465             throw new DoNotRetryIOException(
466                 "Missing required field: qualifer value");
467           }
468           byte[] value = qv.getValue().toByteArray();
469           long ts = timestamp;
470           if (qv.hasTimestamp()) {
471             ts = qv.getTimestamp();
472           }
473           put.add(family, qualifier, ts, value);
474         }
475       }
476     }
477     put.setDurability(toDurability(proto.getDurability()));
478     for (NameBytesPair attribute: proto.getAttributeList()) {
479       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
480     }
481     return put;
482   }
483 
484   /**
485    * Convert a protocol buffer Mutate to a Delete
486    *
487    * @param proto the protocol buffer Mutate to convert
488    * @return the converted client Delete
489    * @throws IOException
490    */
491   public static Delete toDelete(final MutationProto proto)
492   throws IOException {
493     return toDelete(proto, null);
494   }
495 
496   /**
497    * Convert a protocol buffer Mutate to a Delete
498    *
499    * @param proto the protocol buffer Mutate to convert
500    * @param cellScanner if non-null, the data that goes with this delete.
501    * @return the converted client Delete
502    * @throws IOException
503    */
504   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
505   throws IOException {
506     MutationType type = proto.getMutateType();
507     assert type == MutationType.DELETE : type.name();
508     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
509     long timestamp = HConstants.LATEST_TIMESTAMP;
510     if (proto.hasTimestamp()) {
511       timestamp = proto.getTimestamp();
512     }
513     Delete delete = null;
514     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
515     if (cellCount > 0) {
516       // The proto has metadata only and the data is separate to be found in the cellScanner.
517       if (cellScanner == null) {
518         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
519         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
520           TextFormat.shortDebugString(proto));
521       }
522       for (int i = 0; i < cellCount; i++) {
523         if (!cellScanner.advance()) {
524           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
525           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
526             " no cell returned: " + TextFormat.shortDebugString(proto));
527         }
528         Cell cell = cellScanner.current();
529         if (delete == null) {
530           delete =
531             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
532         }
533         delete.addDeleteMarker(KeyValueUtil.ensureKeyValue(cell));
534       }
535     } else {
536       delete = new Delete(row, timestamp);
537       for (ColumnValue column: proto.getColumnValueList()) {
538         byte[] family = column.getFamily().toByteArray();
539         for (QualifierValue qv: column.getQualifierValueList()) {
540           DeleteType deleteType = qv.getDeleteType();
541           byte[] qualifier = null;
542           if (qv.hasQualifier()) {
543             qualifier = qv.getQualifier().toByteArray();
544           }
545           long ts = HConstants.LATEST_TIMESTAMP;
546           if (qv.hasTimestamp()) {
547             ts = qv.getTimestamp();
548           }
549           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
550             delete.deleteColumn(family, qualifier, ts);
551           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
552             delete.deleteColumns(family, qualifier, ts);
553           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
554             delete.deleteFamilyVersion(family, ts);
555           } else {
556             delete.deleteFamily(family, ts);
557           }
558         }
559       }
560     }
561     delete.setDurability(toDurability(proto.getDurability()));
562     for (NameBytesPair attribute: proto.getAttributeList()) {
563       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
564     }
565     return delete;
566   }
567 
568   /**
569    * Convert a protocol buffer Mutate to an Append
570    * @param cellScanner
571    * @param proto the protocol buffer Mutate to convert
572    * @return the converted client Append
573    * @throws IOException 
574    */
575   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
576   throws IOException {
577     MutationType type = proto.getMutateType();
578     assert type == MutationType.APPEND : type.name();
579     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
580     Append append = null;
581     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
582     if (cellCount > 0) {
583       // The proto has metadata only and the data is separate to be found in the cellScanner.
584       if (cellScanner == null) {
585         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
586           toShortString(proto));
587       }
588       for (int i = 0; i < cellCount; i++) {
589         if (!cellScanner.advance()) {
590           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
591             " no cell returned: " + toShortString(proto));
592         }
593         Cell cell = cellScanner.current();
594         if (append == null) {
595           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
596         }
597         append.add(KeyValueUtil.ensureKeyValue(cell));
598       }
599     } else {
600       append = new Append(row);
601       for (ColumnValue column: proto.getColumnValueList()) {
602         byte[] family = column.getFamily().toByteArray();
603         for (QualifierValue qv: column.getQualifierValueList()) {
604           byte[] qualifier = qv.getQualifier().toByteArray();
605           if (!qv.hasValue()) {
606             throw new DoNotRetryIOException(
607               "Missing required field: qualifer value");
608           }
609           byte[] value = qv.getValue().toByteArray();
610           append.add(family, qualifier, value);
611         }
612       }
613     }
614     append.setDurability(toDurability(proto.getDurability()));
615     for (NameBytesPair attribute: proto.getAttributeList()) {
616       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
617     }
618     return append;
619   }
620 
621   /**
622    * Convert a MutateRequest to Mutation
623    *
624    * @param proto the protocol buffer Mutate to convert
625    * @return the converted Mutation
626    * @throws IOException
627    */
628   public static Mutation toMutation(final MutationProto proto) throws IOException {
629     MutationType type = proto.getMutateType();
630     if (type == MutationType.APPEND) {
631       return toAppend(proto, null);
632     }
633     if (type == MutationType.DELETE) {
634       return toDelete(proto, null);
635     }
636     if (type == MutationType.PUT) {
637       return toPut(proto, null);
638     }
639     throw new IOException("Unknown mutation type " + type);
640   }
641 
642   /**
643    * Convert a protocol buffer Mutate to an Increment
644    *
645    * @param proto the protocol buffer Mutate to convert
646    * @return the converted client Increment
647    * @throws IOException
648    */
649   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
650   throws IOException {
651     MutationType type = proto.getMutateType();
652     assert type == MutationType.INCREMENT : type.name();
653     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
654     Increment increment = null;
655     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
656     if (cellCount > 0) {
657       // The proto has metadata only and the data is separate to be found in the cellScanner.
658       if (cellScanner == null) {
659         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
660           TextFormat.shortDebugString(proto));
661       }
662       for (int i = 0; i < cellCount; i++) {
663         if (!cellScanner.advance()) {
664           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
665             " no cell returned: " + TextFormat.shortDebugString(proto));
666         }
667         Cell cell = cellScanner.current();
668         if (increment == null) {
669           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
670         }
671         increment.add(KeyValueUtil.ensureKeyValue(cell));
672       }
673     } else {
674       increment = new Increment(row);
675       for (ColumnValue column: proto.getColumnValueList()) {
676         byte[] family = column.getFamily().toByteArray();
677         for (QualifierValue qv: column.getQualifierValueList()) {
678           byte[] qualifier = qv.getQualifier().toByteArray();
679           if (!qv.hasValue()) {
680             throw new DoNotRetryIOException("Missing required field: qualifer value");
681           }
682           long value = Bytes.toLong(qv.getValue().toByteArray());
683           increment.addColumn(family, qualifier, value);
684         }
685       }
686     }
687     if (proto.hasTimeRange()) {
688       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
689       long minStamp = 0;
690       long maxStamp = Long.MAX_VALUE;
691       if (timeRange.hasFrom()) {
692         minStamp = timeRange.getFrom();
693       }
694       if (timeRange.hasTo()) {
695         maxStamp = timeRange.getTo();
696       }
697       increment.setTimeRange(minStamp, maxStamp);
698     }
699     increment.setDurability(toDurability(proto.getDurability()));
700     return increment;
701   }
702 
703   /**
704    * Convert a client Scan to a protocol buffer Scan
705    *
706    * @param scan the client Scan to convert
707    * @return the converted protocol buffer Scan
708    * @throws IOException
709    */
710   public static ClientProtos.Scan toScan(
711       final Scan scan) throws IOException {
712     ClientProtos.Scan.Builder scanBuilder =
713       ClientProtos.Scan.newBuilder();
714     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
715     if (scan.getBatch() > 0) {
716       scanBuilder.setBatchSize(scan.getBatch());
717     }
718     if (scan.getMaxResultSize() > 0) {
719       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
720     }
721     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
722     if (loadColumnFamiliesOnDemand != null) {
723       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
724     }
725     scanBuilder.setMaxVersions(scan.getMaxVersions());
726     TimeRange timeRange = scan.getTimeRange();
727     if (!timeRange.isAllTime()) {
728       HBaseProtos.TimeRange.Builder timeRangeBuilder =
729         HBaseProtos.TimeRange.newBuilder();
730       timeRangeBuilder.setFrom(timeRange.getMin());
731       timeRangeBuilder.setTo(timeRange.getMax());
732       scanBuilder.setTimeRange(timeRangeBuilder.build());
733     }
734     Map<String, byte[]> attributes = scan.getAttributesMap();
735     if (!attributes.isEmpty()) {
736       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
737       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
738         attributeBuilder.setName(attribute.getKey());
739         attributeBuilder.setValue(ByteString.copyFrom(attribute.getValue()));
740         scanBuilder.addAttribute(attributeBuilder.build());
741       }
742     }
743     byte[] startRow = scan.getStartRow();
744     if (startRow != null && startRow.length > 0) {
745       scanBuilder.setStartRow(ByteString.copyFrom(startRow));
746     }
747     byte[] stopRow = scan.getStopRow();
748     if (stopRow != null && stopRow.length > 0) {
749       scanBuilder.setStopRow(ByteString.copyFrom(stopRow));
750     }
751     if (scan.hasFilter()) {
752       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
753     }
754     if (scan.hasFamilies()) {
755       Column.Builder columnBuilder = Column.newBuilder();
756       for (Map.Entry<byte[],NavigableSet<byte []>>
757           family: scan.getFamilyMap().entrySet()) {
758         columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
759         NavigableSet<byte []> qualifiers = family.getValue();
760         columnBuilder.clearQualifier();
761         if (qualifiers != null && qualifiers.size() > 0) {
762           for (byte [] qualifier: qualifiers) {
763             columnBuilder.addQualifier(ByteString.copyFrom(qualifier));
764           }
765         }
766         scanBuilder.addColumn(columnBuilder.build());
767       }
768     }
769     if (scan.getMaxResultsPerColumnFamily() >= 0) {
770       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
771     }
772     if (scan.getRowOffsetPerColumnFamily() > 0) {
773       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
774     }
775     return scanBuilder.build();
776   }
777 
778   /**
779    * Convert a protocol buffer Scan to a client Scan
780    *
781    * @param proto the protocol buffer Scan to convert
782    * @return the converted client Scan
783    * @throws IOException
784    */
785   public static Scan toScan(
786       final ClientProtos.Scan proto) throws IOException {
787     byte [] startRow = HConstants.EMPTY_START_ROW;
788     byte [] stopRow  = HConstants.EMPTY_END_ROW;
789     if (proto.hasStartRow()) {
790       startRow = proto.getStartRow().toByteArray();
791     }
792     if (proto.hasStopRow()) {
793       stopRow = proto.getStopRow().toByteArray();
794     }
795     Scan scan = new Scan(startRow, stopRow);
796     if (proto.hasCacheBlocks()) {
797       scan.setCacheBlocks(proto.getCacheBlocks());
798     }
799     if (proto.hasMaxVersions()) {
800       scan.setMaxVersions(proto.getMaxVersions());
801     }
802     if (proto.hasStoreLimit()) {
803       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
804     }
805     if (proto.hasStoreOffset()) {
806       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
807     }
808     if (proto.hasLoadColumnFamiliesOnDemand()) {
809       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
810     }
811     if (proto.hasTimeRange()) {
812       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
813       long minStamp = 0;
814       long maxStamp = Long.MAX_VALUE;
815       if (timeRange.hasFrom()) {
816         minStamp = timeRange.getFrom();
817       }
818       if (timeRange.hasTo()) {
819         maxStamp = timeRange.getTo();
820       }
821       scan.setTimeRange(minStamp, maxStamp);
822     }
823     if (proto.hasFilter()) {
824       FilterProtos.Filter filter = proto.getFilter();
825       scan.setFilter(ProtobufUtil.toFilter(filter));
826     }
827     if (proto.hasBatchSize()) {
828       scan.setBatch(proto.getBatchSize());
829     }
830     if (proto.hasMaxResultSize()) {
831       scan.setMaxResultSize(proto.getMaxResultSize());
832     }
833     for (NameBytesPair attribute: proto.getAttributeList()) {
834       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
835     }
836     if (proto.getColumnCount() > 0) {
837       for (Column column: proto.getColumnList()) {
838         byte[] family = column.getFamily().toByteArray();
839         if (column.getQualifierCount() > 0) {
840           for (ByteString qualifier: column.getQualifierList()) {
841             scan.addColumn(family, qualifier.toByteArray());
842           }
843         } else {
844           scan.addFamily(family);
845         }
846       }
847     }
848     return scan;
849   }
850 
851   /**
852    * Create a protocol buffer Get based on a client Get.
853    *
854    * @param get the client Get
855    * @return a protocol buffer Get
856    * @throws IOException
857    */
858   public static ClientProtos.Get toGet(
859       final Get get) throws IOException {
860     ClientProtos.Get.Builder builder =
861       ClientProtos.Get.newBuilder();
862     builder.setRow(ByteString.copyFrom(get.getRow()));
863     builder.setCacheBlocks(get.getCacheBlocks());
864     builder.setMaxVersions(get.getMaxVersions());
865     if (get.getFilter() != null) {
866       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
867     }
868     TimeRange timeRange = get.getTimeRange();
869     if (!timeRange.isAllTime()) {
870       HBaseProtos.TimeRange.Builder timeRangeBuilder =
871         HBaseProtos.TimeRange.newBuilder();
872       timeRangeBuilder.setFrom(timeRange.getMin());
873       timeRangeBuilder.setTo(timeRange.getMax());
874       builder.setTimeRange(timeRangeBuilder.build());
875     }
876     Map<String, byte[]> attributes = get.getAttributesMap();
877     if (!attributes.isEmpty()) {
878       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
879       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
880         attributeBuilder.setName(attribute.getKey());
881         attributeBuilder.setValue(ByteString.copyFrom(attribute.getValue()));
882         builder.addAttribute(attributeBuilder.build());
883       }
884     }
885     if (get.hasFamilies()) {
886       Column.Builder columnBuilder = Column.newBuilder();
887       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
888       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
889         NavigableSet<byte[]> qualifiers = family.getValue();
890         columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
891         columnBuilder.clearQualifier();
892         if (qualifiers != null && qualifiers.size() > 0) {
893           for (byte[] qualifier: qualifiers) {
894             columnBuilder.addQualifier(ByteString.copyFrom(qualifier));
895           }
896         }
897         builder.addColumn(columnBuilder.build());
898       }
899     }
900     if (get.getMaxResultsPerColumnFamily() >= 0) {
901       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
902     }
903     if (get.getRowOffsetPerColumnFamily() > 0) {
904       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
905     }
906     return builder.build();
907   }
908 
909   /**
910    * Convert a client Increment to a protobuf Mutate.
911    *
912    * @param increment
913    * @return the converted mutate
914    */
915   public static MutationProto toMutation(final Increment increment) {
916     MutationProto.Builder builder = MutationProto.newBuilder();
917     builder.setRow(ByteString.copyFrom(increment.getRow()));
918     builder.setMutateType(MutationType.INCREMENT);
919     builder.setDurability(toDurability(increment.getDurability()));
920     TimeRange timeRange = increment.getTimeRange();
921     if (!timeRange.isAllTime()) {
922       HBaseProtos.TimeRange.Builder timeRangeBuilder =
923         HBaseProtos.TimeRange.newBuilder();
924       timeRangeBuilder.setFrom(timeRange.getMin());
925       timeRangeBuilder.setTo(timeRange.getMax());
926       builder.setTimeRange(timeRangeBuilder.build());
927     }
928     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
929     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
930    for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
931       columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
932       columnBuilder.clearQualifierValue();
933       List<Cell> values = family.getValue();
934       if (values != null && values.size() > 0) {
935         for (Cell cell: values) {
936           KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
937           valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier()));
938           valueBuilder.setValue(ByteString.copyFrom(kv.getValue()));
939           columnBuilder.addQualifierValue(valueBuilder.build());
940         }
941       }
942       builder.addColumnValue(columnBuilder.build());
943     }
944     return builder.build();
945   }
946 
947   /**
948    * Create a protocol buffer Mutate based on a client Mutation
949    *
950    * @param type
951    * @param mutation
952    * @return a protobuf'd Mutation
953    * @throws IOException
954    */
955   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
956   throws IOException {
957     MutationProto.Builder builder = getMutationBuilderAndSetCommonFields(type, mutation);
958     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
959     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
960     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
961       columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
962       columnBuilder.clearQualifierValue();
963       for (Cell cell: family.getValue()) {
964         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
965         valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier()));
966         valueBuilder.setValue(ByteString.copyFrom(kv.getValue()));
967         valueBuilder.setTimestamp(kv.getTimestamp());
968         if (type == MutationType.DELETE) {
969           KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
970           valueBuilder.setDeleteType(toDeleteType(keyValueType));
971         }
972         columnBuilder.addQualifierValue(valueBuilder.build());
973       }
974       builder.addColumnValue(columnBuilder.build());
975     }
976     return builder.build();
977   }
978 
979   /**
980    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
981    * Understanding is that the Cell will be transported other than via protobuf.
982    * @param type
983    * @param mutation
984    * @return a protobuf'd Mutation
985    * @throws IOException
986    */
987   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
988   throws IOException {
989     MutationProto.Builder builder = getMutationBuilderAndSetCommonFields(type, mutation);
990     builder.setAssociatedCellCount(mutation.size());
991     return builder.build();
992   }
993 
994   /**
995    * Code shared by {@link #toMutation(MutationType, Mutation)} and
996    * {@link #toMutationNoData(MutationType, Mutation)}
997    * @param type
998    * @param mutation
999    * @return A partly-filled out protobuf'd Mutation.
1000    */
1001   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1002       final Mutation mutation) {
1003     MutationProto.Builder builder = MutationProto.newBuilder();
1004     builder.setRow(ByteString.copyFrom(mutation.getRow()));
1005     builder.setMutateType(type);
1006     builder.setDurability(toDurability(mutation.getDurability()));
1007     builder.setTimestamp(mutation.getTimeStamp());
1008     Map<String, byte[]> attributes = mutation.getAttributesMap();
1009     if (!attributes.isEmpty()) {
1010       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1011       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1012         attributeBuilder.setName(attribute.getKey());
1013         attributeBuilder.setValue(ByteString.copyFrom(attribute.getValue()));
1014         builder.addAttribute(attributeBuilder.build());
1015       }
1016     }
1017     return builder;
1018   }
1019 
1020   /**
1021    * Convert a client Result to a protocol buffer Result
1022    *
1023    * @param result the client Result to convert
1024    * @return the converted protocol buffer Result
1025    */
1026   public static ClientProtos.Result toResult(final Result result) {
1027     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1028     Cell [] cells = result.raw();
1029     if (cells != null) {
1030       for (Cell c : cells) {
1031         builder.addCell(toCell(c));
1032       }
1033     }
1034     return builder.build();
1035   }
1036 
1037   /**
1038    * Convert a client Result to a protocol buffer Result.
1039    * The pb Result does not include the Cell data.  That is for transport otherwise.
1040    *
1041    * @param result the client Result to convert
1042    * @return the converted protocol buffer Result
1043    */
1044   public static ClientProtos.Result toResultNoData(final Result result) {
1045     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1046     builder.setAssociatedCellCount(result.size());
1047     return builder.build();
1048   }
1049 
1050   /**
1051    * Convert a protocol buffer Result to a client Result
1052    *
1053    * @param proto the protocol buffer Result to convert
1054    * @return the converted client Result
1055    */
1056   public static Result toResult(final ClientProtos.Result proto) {
1057     List<CellProtos.Cell> values = proto.getCellList();
1058     List<Cell> cells = new ArrayList<Cell>(values.size());
1059     for (CellProtos.Cell c: values) {
1060       cells.add(toCell(c));
1061     }
1062     return new Result(cells);
1063   }
1064 
1065   /**
1066    * Convert a protocol buffer Result to a client Result
1067    *
1068    * @param proto the protocol buffer Result to convert
1069    * @param scanner Optional cell scanner.
1070    * @return the converted client Result
1071    * @throws IOException
1072    */
1073   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1074   throws IOException {
1075     // TODO: Unit test that has some Cells in scanner and some in the proto.
1076     List<Cell> cells = null;
1077     if (proto.hasAssociatedCellCount()) {
1078       int count = proto.getAssociatedCellCount();
1079       cells = new ArrayList<Cell>(count);
1080       for (int i = 0; i < count; i++) {
1081         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1082         cells.add(scanner.current());
1083       }
1084     }
1085     List<CellProtos.Cell> values = proto.getCellList();
1086     if (cells == null) cells = new ArrayList<Cell>(values.size());
1087     for (CellProtos.Cell c: values) {
1088       cells.add(toCell(c));
1089     }
1090     return new Result(cells);
1091   }
1092 
1093   /**
1094    * Convert a ByteArrayComparable to a protocol buffer Comparator
1095    *
1096    * @param comparator the ByteArrayComparable to convert
1097    * @return the converted protocol buffer Comparator
1098    */
1099   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1100     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1101     builder.setName(comparator.getClass().getName());
1102     builder.setSerializedComparator(ByteString.copyFrom(comparator.toByteArray()));
1103     return builder.build();
1104   }
1105 
1106   /**
1107    * Convert a protocol buffer Comparator to a ByteArrayComparable
1108    *
1109    * @param proto the protocol buffer Comparator to convert
1110    * @return the converted ByteArrayComparable
1111    */
1112   @SuppressWarnings("unchecked")
1113   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1114   throws IOException {
1115     String type = proto.getName();
1116     String funcName = "parseFrom";
1117     byte [] value = proto.getSerializedComparator().toByteArray();
1118     try {
1119       Class<? extends ByteArrayComparable> c =
1120         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1121       Method parseFrom = c.getMethod(funcName, byte[].class);
1122       if (parseFrom == null) {
1123         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1124       }
1125       return (ByteArrayComparable)parseFrom.invoke(null, value);
1126     } catch (Exception e) {
1127       throw new IOException(e);
1128     }
1129   }
1130 
1131   /**
1132    * Convert a protocol buffer Filter to a client Filter
1133    *
1134    * @param proto the protocol buffer Filter to convert
1135    * @return the converted Filter
1136    */
1137   @SuppressWarnings("unchecked")
1138   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1139     String type = proto.getName();
1140     final byte [] value = proto.getSerializedFilter().toByteArray();
1141     String funcName = "parseFrom";
1142     try {
1143       Class<? extends Filter> c =
1144         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1145       Method parseFrom = c.getMethod(funcName, byte[].class);
1146       if (parseFrom == null) {
1147         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1148       }
1149       return (Filter)parseFrom.invoke(c, value);
1150     } catch (Exception e) {
1151       throw new IOException(e);
1152     }
1153   }
1154 
1155   /**
1156    * Convert a client Filter to a protocol buffer Filter
1157    *
1158    * @param filter the Filter to convert
1159    * @return the converted protocol buffer Filter
1160    */
1161   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1162     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1163     builder.setName(filter.getClass().getName());
1164     builder.setSerializedFilter(ByteString.copyFrom(filter.toByteArray()));
1165     return builder.build();
1166   }
1167 
1168   /**
1169    * Convert a delete KeyValue type to protocol buffer DeleteType.
1170    *
1171    * @param type
1172    * @return protocol buffer DeleteType
1173    * @throws IOException
1174    */
1175   public static DeleteType toDeleteType(
1176       KeyValue.Type type) throws IOException {
1177     switch (type) {
1178     case Delete:
1179       return DeleteType.DELETE_ONE_VERSION;
1180     case DeleteColumn:
1181       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1182     case DeleteFamily:
1183       return DeleteType.DELETE_FAMILY;
1184     case DeleteFamilyVersion:
1185       return DeleteType.DELETE_FAMILY_VERSION;
1186     default:
1187         throw new IOException("Unknown delete type: " + type);
1188     }
1189   }
1190 
1191   /**
1192    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1193    *
1194    * @param parameter the protocol buffer Parameter to convert
1195    * @return the converted Exception
1196    * @throws IOException if failed to deserialize the parameter
1197    */
1198   @SuppressWarnings("unchecked")
1199   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1200     if (parameter == null || !parameter.hasValue()) return null;
1201     String desc = parameter.getValue().toStringUtf8();
1202     String type = parameter.getName();
1203     try {
1204       Class<? extends Throwable> c =
1205         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1206       Constructor<? extends Throwable> cn = null;
1207       try {
1208         cn = c.getDeclaredConstructor(String.class);
1209         return cn.newInstance(desc);
1210       } catch (NoSuchMethodException e) {
1211         // Could be a raw RemoteException. See HBASE-8987.
1212         cn = c.getDeclaredConstructor(String.class, String.class);
1213         return cn.newInstance(type, desc);
1214       }
1215     } catch (Exception e) {
1216       throw new IOException(e);
1217     }
1218   }
1219 
1220 // Start helpers for Client
1221 
1222   /**
1223    * A helper to invoke a Get using client protocol.
1224    *
1225    * @param client
1226    * @param regionName
1227    * @param get
1228    * @return the result of the Get
1229    * @throws IOException
1230    */
1231   public static Result get(final ClientService.BlockingInterface client,
1232       final byte[] regionName, final Get get) throws IOException {
1233     GetRequest request =
1234       RequestConverter.buildGetRequest(regionName, get);
1235     try {
1236       GetResponse response = client.get(null, request);
1237       if (response == null) return null;
1238       return toResult(response.getResult());
1239     } catch (ServiceException se) {
1240       throw getRemoteException(se);
1241     }
1242   }
1243 
1244   /**
1245    * A helper to get a row of the closet one before using client protocol.
1246    *
1247    * @param client
1248    * @param regionName
1249    * @param row
1250    * @param family
1251    * @return the row or the closestRowBefore if it doesn't exist
1252    * @throws IOException
1253    */
1254   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1255       final byte[] regionName, final byte[] row,
1256       final byte[] family) throws IOException {
1257     GetRequest request =
1258       RequestConverter.buildGetRowOrBeforeRequest(
1259         regionName, row, family);
1260     try {
1261       GetResponse response = client.get(null, request);
1262       if (!response.hasResult()) return null;
1263       return toResult(response.getResult());
1264     } catch (ServiceException se) {
1265       throw getRemoteException(se);
1266     }
1267   }
1268 
1269   /**
1270    * A helper to bulk load a list of HFiles using client protocol.
1271    *
1272    * @param client
1273    * @param familyPaths
1274    * @param regionName
1275    * @param assignSeqNum
1276    * @return true if all are loaded
1277    * @throws IOException
1278    */
1279   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1280       final List<Pair<byte[], String>> familyPaths,
1281       final byte[] regionName, boolean assignSeqNum) throws IOException {
1282     BulkLoadHFileRequest request =
1283       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1284     try {
1285       BulkLoadHFileResponse response =
1286         client.bulkLoadHFile(null, request);
1287       return response.getLoaded();
1288     } catch (ServiceException se) {
1289       throw getRemoteException(se);
1290     }
1291   }
1292 
1293   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1294       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1295     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1296         .setCall(call).setRegion(
1297             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1298     try {
1299       CoprocessorServiceResponse response =
1300           client.execService(null, request);
1301       return response;
1302     } catch (ServiceException se) {
1303       throw getRemoteException(se);
1304     }
1305   }
1306 
1307   public static CoprocessorServiceResponse execService(
1308     final MasterAdminService.BlockingInterface client, final CoprocessorServiceCall call)
1309   throws IOException {
1310     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1311         .setCall(call).setRegion(
1312             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1313     try {
1314       CoprocessorServiceResponse response =
1315           client.execMasterService(null, request);
1316       return response;
1317     } catch (ServiceException se) {
1318       throw getRemoteException(se);
1319     }
1320   }
1321 
1322   @SuppressWarnings("unchecked")
1323   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1324       throws Exception {
1325     return (T)Methods.call(service, null, "newStub",
1326         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1327   }
1328 
1329 // End helpers for Client
1330 // Start helpers for Admin
1331 
1332   /**
1333    * A helper to retrieve region info given a region name
1334    * using admin protocol.
1335    *
1336    * @param admin
1337    * @param regionName
1338    * @return the retrieved region info
1339    * @throws IOException
1340    */
1341   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1342       final byte[] regionName) throws IOException {
1343     try {
1344       GetRegionInfoRequest request =
1345         RequestConverter.buildGetRegionInfoRequest(regionName);
1346       GetRegionInfoResponse response =
1347         admin.getRegionInfo(null, request);
1348       return HRegionInfo.convert(response.getRegionInfo());
1349     } catch (ServiceException se) {
1350       throw getRemoteException(se);
1351     }
1352   }
1353 
1354   /**
1355    * A helper to close a region given a region name
1356    * using admin protocol.
1357    *
1358    * @param admin
1359    * @param regionName
1360    * @param transitionInZK
1361    * @throws IOException
1362    */
1363   public static void closeRegion(final AdminService.BlockingInterface admin,
1364       final byte[] regionName, final boolean transitionInZK) throws IOException {
1365     CloseRegionRequest closeRegionRequest =
1366       RequestConverter.buildCloseRegionRequest(regionName, transitionInZK);
1367     try {
1368       admin.closeRegion(null, closeRegionRequest);
1369     } catch (ServiceException se) {
1370       throw getRemoteException(se);
1371     }
1372   }
1373 
1374   /**
1375    * A helper to close a region given a region name
1376    * using admin protocol.
1377    *
1378    * @param admin
1379    * @param regionName
1380    * @param versionOfClosingNode
1381    * @return true if the region is closed
1382    * @throws IOException
1383    */
1384   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1385       final byte[] regionName,
1386       final int versionOfClosingNode, final ServerName destinationServer,
1387       final boolean transitionInZK) throws IOException {
1388     CloseRegionRequest closeRegionRequest =
1389       RequestConverter.buildCloseRegionRequest(
1390         regionName, versionOfClosingNode, destinationServer, transitionInZK);
1391     try {
1392       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1393       return ResponseConverter.isClosed(response);
1394     } catch (ServiceException se) {
1395       throw getRemoteException(se);
1396     }
1397   }
1398 
1399 
1400   /**
1401    * A helper to open a region using admin protocol.
1402    * @param admin
1403    * @param region
1404    * @throws IOException
1405    */
1406   public static void openRegion(final AdminService.BlockingInterface admin,
1407       final HRegionInfo region) throws IOException {
1408     OpenRegionRequest request =
1409       RequestConverter.buildOpenRegionRequest(region, -1, null);
1410     try {
1411       admin.openRegion(null, request);
1412     } catch (ServiceException se) {
1413       throw ProtobufUtil.getRemoteException(se);
1414     }
1415   }
1416 
1417   /**
1418    * A helper to get the all the online regions on a region
1419    * server using admin protocol.
1420    *
1421    * @param admin
1422    * @return a list of online region info
1423    * @throws IOException
1424    */
1425   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1426   throws IOException {
1427     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1428     GetOnlineRegionResponse response = null;
1429     try {
1430       response = admin.getOnlineRegion(null, request);
1431     } catch (ServiceException se) {
1432       throw getRemoteException(se);
1433     }
1434     return getRegionInfos(response);
1435   }
1436 
1437   /**
1438    * Get the list of region info from a GetOnlineRegionResponse
1439    *
1440    * @param proto the GetOnlineRegionResponse
1441    * @return the list of region info or null if <code>proto</code> is null
1442    */
1443   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1444     if (proto == null) return null;
1445     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1446     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1447       regionInfos.add(HRegionInfo.convert(regionInfo));
1448     }
1449     return regionInfos;
1450   }
1451 
1452   /**
1453    * A helper to get the info of a region server using admin protocol.
1454    *
1455    * @param admin
1456    * @return the server name
1457    * @throws IOException
1458    */
1459   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1460   throws IOException {
1461     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1462     try {
1463       GetServerInfoResponse response = admin.getServerInfo(null, request);
1464       return response.getServerInfo();
1465     } catch (ServiceException se) {
1466       throw getRemoteException(se);
1467     }
1468   }
1469 
1470   /**
1471    * A helper to get the list of files of a column family
1472    * on a given region using admin protocol.
1473    *
1474    * @param admin
1475    * @param regionName
1476    * @param family
1477    * @return the list of store files
1478    * @throws IOException
1479    */
1480   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1481       final byte[] regionName, final byte[] family)
1482   throws IOException {
1483     GetStoreFileRequest request =
1484       RequestConverter.buildGetStoreFileRequest(regionName, family);
1485     try {
1486       GetStoreFileResponse response = admin.getStoreFile(null, request);
1487       return response.getStoreFileList();
1488     } catch (ServiceException se) {
1489       throw ProtobufUtil.getRemoteException(se);
1490     }
1491   }
1492 
1493   /**
1494    * A helper to split a region using admin protocol.
1495    *
1496    * @param admin
1497    * @param hri
1498    * @param splitPoint
1499    * @throws IOException
1500    */
1501   public static void split(final AdminService.BlockingInterface admin,
1502       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1503     SplitRegionRequest request =
1504       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1505     try {
1506       admin.splitRegion(null, request);
1507     } catch (ServiceException se) {
1508       throw ProtobufUtil.getRemoteException(se);
1509     }
1510   }
1511 
1512   /**
1513    * A helper to merge regions using admin protocol. Send request to
1514    * regionserver.
1515    * @param admin
1516    * @param region_a
1517    * @param region_b
1518    * @param forcible true if do a compulsory merge, otherwise we will only merge
1519    *          two adjacent regions
1520    * @throws IOException
1521    */
1522   public static void mergeRegions(final AdminService.BlockingInterface admin,
1523       final HRegionInfo region_a, final HRegionInfo region_b,
1524       final boolean forcible) throws IOException {
1525     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1526         region_a.getRegionName(), region_b.getRegionName(),forcible);
1527     try {
1528       admin.mergeRegions(null, request);
1529     } catch (ServiceException se) {
1530       throw ProtobufUtil.getRemoteException(se);
1531     }
1532   }
1533 
1534 // End helpers for Admin
1535 
1536   /*
1537    * Get the total (read + write) requests from a RegionLoad pb
1538    * @param rl - RegionLoad pb
1539    * @return total (read + write) requests
1540    */
1541   public static long getTotalRequestsCount(RegionLoad rl) {
1542     if (rl == null) {
1543       return 0;
1544     }
1545 
1546     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1547   }
1548 
1549 
1550   /**
1551    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1552    */
1553   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1554     // Allocate arbitrary big size so we avoid resizing.
1555     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1556     m.writeDelimitedTo(baos);
1557     baos.close();
1558     return ProtobufUtil.prependPBMagic(baos.toByteArray());
1559   }
1560 
1561   /**
1562    * Converts a Permission proto to a client Permission object.
1563    *
1564    * @param proto the protobuf Permission
1565    * @return the converted Permission
1566    */
1567   public static Permission toPermission(AccessControlProtos.Permission proto) {
1568     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1569       return toTablePermission(proto);
1570     } else {
1571       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1572       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1573     }
1574   }
1575 
1576   /**
1577    * Converts a Permission proto to a client TablePermission object.
1578    *
1579    * @param proto the protobuf Permission
1580    * @return the converted TablePermission
1581    */
1582   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1583     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1584       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1585       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1586 
1587       return new TablePermission(null, null, null,
1588           actions.toArray(new Permission.Action[actions.size()]));
1589     }
1590     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1591       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1592       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1593 
1594       if(!proto.hasNamespacePermission()) {
1595         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1596       }
1597       String namespace = perm.getNamespaceName().toStringUtf8();
1598       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1599     }
1600     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1601       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1602       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1603 
1604       byte[] qualifier = null;
1605       byte[] family = null;
1606       TableName table = null;
1607 
1608       if (!perm.hasTableName()) {
1609         throw new IllegalStateException("TableName cannot be empty");
1610       }
1611       table = ProtobufUtil.toTableName(perm.getTableName());
1612 
1613       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1614       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1615 
1616       return new TablePermission(table, family, qualifier,
1617           actions.toArray(new Permission.Action[actions.size()]));
1618     }
1619     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1620   }
1621 
1622   /**
1623    * Convert a client Permission to a Permission proto
1624    *
1625    * @param perm the client Permission
1626    * @return the protobuf Permission
1627    */
1628   public static AccessControlProtos.Permission toPermission(Permission perm) {
1629     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1630     if (perm instanceof TablePermission) {
1631       TablePermission tablePerm = (TablePermission)perm;
1632       if(tablePerm.hasNamespace()) {
1633         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1634 
1635         AccessControlProtos.NamespacePermission.Builder builder =
1636             AccessControlProtos.NamespacePermission.newBuilder();
1637         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1638         for (Permission.Action a : perm.getActions()) {
1639           builder.addAction(toPermissionAction(a));
1640         }
1641         ret.setNamespacePermission(builder);
1642       } else if (tablePerm.hasTable()) {
1643         ret.setType(AccessControlProtos.Permission.Type.Table);
1644 
1645         AccessControlProtos.TablePermission.Builder builder =
1646             AccessControlProtos.TablePermission.newBuilder();
1647         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTable()));
1648         if (tablePerm.hasFamily()) {
1649           builder.setFamily(ByteString.copyFrom(tablePerm.getFamily()));
1650         }
1651         if (tablePerm.hasQualifier()) {
1652           builder.setQualifier(ByteString.copyFrom(tablePerm.getQualifier()));
1653         }
1654         for (Permission.Action a : perm.getActions()) {
1655           builder.addAction(toPermissionAction(a));
1656         }
1657         ret.setTablePermission(builder);
1658       }
1659     } else {
1660       ret.setType(AccessControlProtos.Permission.Type.Global);
1661 
1662       AccessControlProtos.GlobalPermission.Builder builder =
1663           AccessControlProtos.GlobalPermission.newBuilder();
1664       for (Permission.Action a : perm.getActions()) {
1665         builder.addAction(toPermissionAction(a));
1666       }
1667       ret.setGlobalPermission(builder);
1668     }
1669     return ret.build();
1670   }
1671 
1672   /**
1673    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
1674    *
1675    * @param protoActions the list of protobuf Actions
1676    * @return the converted list of Actions
1677    */
1678   public static List<Permission.Action> toPermissionActions(
1679       List<AccessControlProtos.Permission.Action> protoActions) {
1680     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
1681     for (AccessControlProtos.Permission.Action a : protoActions) {
1682       actions.add(toPermissionAction(a));
1683     }
1684     return actions;
1685   }
1686 
1687   /**
1688    * Converts a Permission.Action proto to a client Permission.Action object.
1689    *
1690    * @param action the protobuf Action
1691    * @return the converted Action
1692    */
1693   public static Permission.Action toPermissionAction(
1694       AccessControlProtos.Permission.Action action) {
1695     switch (action) {
1696       case READ:
1697         return Permission.Action.READ;
1698       case WRITE:
1699         return Permission.Action.WRITE;
1700       case EXEC:
1701         return Permission.Action.EXEC;
1702       case CREATE:
1703         return Permission.Action.CREATE;
1704       case ADMIN:
1705         return Permission.Action.ADMIN;
1706     }
1707     throw new IllegalArgumentException("Unknown action value "+action.name());
1708   }
1709 
1710   /**
1711    * Convert a client Permission.Action to a Permission.Action proto
1712    *
1713    * @param action the client Action
1714    * @return the protobuf Action
1715    */
1716   public static AccessControlProtos.Permission.Action toPermissionAction(
1717       Permission.Action action) {
1718     switch (action) {
1719       case READ:
1720         return AccessControlProtos.Permission.Action.READ;
1721       case WRITE:
1722         return AccessControlProtos.Permission.Action.WRITE;
1723       case EXEC:
1724         return AccessControlProtos.Permission.Action.EXEC;
1725       case CREATE:
1726         return AccessControlProtos.Permission.Action.CREATE;
1727       case ADMIN:
1728         return AccessControlProtos.Permission.Action.ADMIN;
1729     }
1730     throw new IllegalArgumentException("Unknown action value "+action.name());
1731   }
1732 
1733   /**
1734    * Convert a client user permission to a user permission proto
1735    *
1736    * @param perm the client UserPermission
1737    * @return the protobuf UserPermission
1738    */
1739   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
1740     return AccessControlProtos.UserPermission.newBuilder()
1741         .setUser(ByteString.copyFrom(perm.getUser()))
1742         .setPermission(toPermission(perm))
1743         .build();
1744   }
1745 
1746   /**
1747    * Converts a user permission proto to a client user permission object.
1748    *
1749    * @param proto the protobuf UserPermission
1750    * @return the converted UserPermission
1751    */
1752   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
1753     return new UserPermission(proto.getUser().toByteArray(),
1754         toTablePermission(proto.getPermission()));
1755   }
1756 
1757   /**
1758    * Convert a ListMultimap<String, TablePermission> where key is username
1759    * to a protobuf UserPermission
1760    *
1761    * @param perm the list of user and table permissions
1762    * @return the protobuf UserTablePermissions
1763    */
1764   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
1765       ListMultimap<String, TablePermission> perm) {
1766     AccessControlProtos.UsersAndPermissions.Builder builder =
1767                   AccessControlProtos.UsersAndPermissions.newBuilder();
1768     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
1769       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
1770                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
1771       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
1772       for (TablePermission tablePerm: entry.getValue()) {
1773         userPermBuilder.addPermissions(toPermission(tablePerm));
1774       }
1775       builder.addUserPermissions(userPermBuilder.build());
1776     }
1777     return builder.build();
1778   }
1779 
1780   /**
1781    * A utility used to grant a user global permissions.
1782    * <p>
1783    * It's also called by the shell, in case you want to find references.
1784    *
1785    * @param protocol the AccessControlService protocol proxy
1786    * @param userShortName the short name of the user to grant permissions
1787    * @param actions the permissions to be granted
1788    * @throws ServiceException
1789    */
1790   public static void grant(AccessControlService.BlockingInterface protocol,
1791       String userShortName, Permission.Action... actions) throws ServiceException {
1792     List<AccessControlProtos.Permission.Action> permActions =
1793         Lists.newArrayListWithCapacity(actions.length);
1794     for (Permission.Action a : actions) {
1795       permActions.add(ProtobufUtil.toPermissionAction(a));
1796     }
1797     AccessControlProtos.GrantRequest request = RequestConverter.
1798       buildGrantRequest(userShortName, permActions.toArray(
1799         new AccessControlProtos.Permission.Action[actions.length]));
1800     protocol.grant(null, request);
1801   }
1802 
1803   /**
1804    * A utility used to grant a user table permissions. The permissions will
1805    * be for a table table/column family/qualifier.
1806    * <p>
1807    * It's also called by the shell, in case you want to find references.
1808    *
1809    * @param protocol the AccessControlService protocol proxy
1810    * @param userShortName the short name of the user to grant permissions
1811    * @param tableName optional table name
1812    * @param f optional column family
1813    * @param q optional qualifier
1814    * @param actions the permissions to be granted
1815    * @throws ServiceException
1816    */
1817   public static void grant(AccessControlService.BlockingInterface protocol,
1818       String userShortName, TableName tableName, byte[] f, byte[] q,
1819       Permission.Action... actions) throws ServiceException {
1820     List<AccessControlProtos.Permission.Action> permActions =
1821         Lists.newArrayListWithCapacity(actions.length);
1822     for (Permission.Action a : actions) {
1823       permActions.add(ProtobufUtil.toPermissionAction(a));
1824     }
1825     AccessControlProtos.GrantRequest request = RequestConverter.
1826       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
1827         new AccessControlProtos.Permission.Action[actions.length]));
1828     protocol.grant(null, request);
1829   }
1830 
1831   /**
1832    * A utility used to grant a user namespace permissions.
1833    * <p>
1834    * It's also called by the shell, in case you want to find references.
1835    *
1836    * @param protocol the AccessControlService protocol proxy
1837    * @param namespace the short name of the user to grant permissions
1838    * @param actions the permissions to be granted
1839    * @throws ServiceException
1840    */
1841   public static void grant(AccessControlService.BlockingInterface protocol,
1842       String userShortName, String namespace,
1843       Permission.Action... actions) throws ServiceException {
1844     List<AccessControlProtos.Permission.Action> permActions =
1845         Lists.newArrayListWithCapacity(actions.length);
1846     for (Permission.Action a : actions) {
1847       permActions.add(ProtobufUtil.toPermissionAction(a));
1848     }
1849     AccessControlProtos.GrantRequest request = RequestConverter.
1850       buildGrantRequest(userShortName, namespace, permActions.toArray(
1851         new AccessControlProtos.Permission.Action[actions.length]));
1852     protocol.grant(null, request);
1853   }
1854 
1855   /**
1856    * A utility used to revoke a user's global permissions.
1857    * <p>
1858    * It's also called by the shell, in case you want to find references.
1859    *
1860    * @param protocol the AccessControlService protocol proxy
1861    * @param userShortName the short name of the user to revoke permissions
1862    * @param actions the permissions to be revoked
1863    * @throws ServiceException
1864    */
1865   public static void revoke(AccessControlService.BlockingInterface protocol,
1866       String userShortName, Permission.Action... actions) throws ServiceException {
1867     List<AccessControlProtos.Permission.Action> permActions =
1868         Lists.newArrayListWithCapacity(actions.length);
1869     for (Permission.Action a : actions) {
1870       permActions.add(ProtobufUtil.toPermissionAction(a));
1871     }
1872     AccessControlProtos.RevokeRequest request = RequestConverter.
1873       buildRevokeRequest(userShortName, permActions.toArray(
1874         new AccessControlProtos.Permission.Action[actions.length]));
1875     protocol.revoke(null, request);
1876   }
1877 
1878   /**
1879    * A utility used to revoke a user's table permissions. The permissions will
1880    * be for a table/column family/qualifier.
1881    * <p>
1882    * It's also called by the shell, in case you want to find references.
1883    *
1884    * @param protocol the AccessControlService protocol proxy
1885    * @param userShortName the short name of the user to revoke permissions
1886    * @param tableName optional table name
1887    * @param f optional column family
1888    * @param q optional qualifier
1889    * @param actions the permissions to be revoked
1890    * @throws ServiceException
1891    */
1892   public static void revoke(AccessControlService.BlockingInterface protocol,
1893       String userShortName, TableName tableName, byte[] f, byte[] q,
1894       Permission.Action... actions) throws ServiceException {
1895     List<AccessControlProtos.Permission.Action> permActions =
1896         Lists.newArrayListWithCapacity(actions.length);
1897     for (Permission.Action a : actions) {
1898       permActions.add(ProtobufUtil.toPermissionAction(a));
1899     }
1900     AccessControlProtos.RevokeRequest request = RequestConverter.
1901       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
1902         new AccessControlProtos.Permission.Action[actions.length]));
1903     protocol.revoke(null, request);
1904   }
1905 
1906   /**
1907    * A utility used to revoke a user's namespace permissions.
1908    * <p>
1909    * It's also called by the shell, in case you want to find references.
1910    *
1911    * @param protocol the AccessControlService protocol proxy
1912    * @param userShortName the short name of the user to revoke permissions
1913    * @param namespace optional table name
1914    * @param actions the permissions to be revoked
1915    * @throws ServiceException
1916    */
1917   public static void revoke(AccessControlService.BlockingInterface protocol,
1918       String userShortName, String namespace,
1919       Permission.Action... actions) throws ServiceException {
1920     List<AccessControlProtos.Permission.Action> permActions =
1921         Lists.newArrayListWithCapacity(actions.length);
1922     for (Permission.Action a : actions) {
1923       permActions.add(ProtobufUtil.toPermissionAction(a));
1924     }
1925     AccessControlProtos.RevokeRequest request = RequestConverter.
1926       buildRevokeRequest(userShortName, namespace, permActions.toArray(
1927         new AccessControlProtos.Permission.Action[actions.length]));
1928     protocol.revoke(null, request);
1929   }
1930 
1931   /**
1932    * A utility used to get user's global permissions.
1933    * <p>
1934    * It's also called by the shell, in case you want to find references.
1935    *
1936    * @param protocol the AccessControlService protocol proxy
1937    * @throws ServiceException
1938    */
1939   public static List<UserPermission> getUserPermissions(
1940       AccessControlService.BlockingInterface protocol) throws ServiceException {
1941     AccessControlProtos.UserPermissionsRequest.Builder builder =
1942       AccessControlProtos.UserPermissionsRequest.newBuilder();
1943     builder.setType(AccessControlProtos.Permission.Type.Global);
1944     AccessControlProtos.UserPermissionsRequest request = builder.build();
1945     AccessControlProtos.UserPermissionsResponse response =
1946       protocol.getUserPermissions(null, request);
1947     List<UserPermission> perms = new ArrayList<UserPermission>();
1948     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
1949       perms.add(ProtobufUtil.toUserPermission(perm));
1950     }
1951     return perms;
1952   }
1953 
1954   /**
1955    * A utility used to get user table permissions.
1956    * <p>
1957    * It's also called by the shell, in case you want to find references.
1958    *
1959    * @param protocol the AccessControlService protocol proxy
1960    * @param t optional table name
1961    * @throws ServiceException
1962    */
1963   public static List<UserPermission> getUserPermissions(
1964       AccessControlService.BlockingInterface protocol,
1965       TableName t) throws ServiceException {
1966     AccessControlProtos.UserPermissionsRequest.Builder builder =
1967       AccessControlProtos.UserPermissionsRequest.newBuilder();
1968     if (t != null) {
1969       builder.setTableName(ProtobufUtil.toProtoTableName(t));
1970     }
1971     builder.setType(AccessControlProtos.Permission.Type.Table);
1972     AccessControlProtos.UserPermissionsRequest request = builder.build();
1973     AccessControlProtos.UserPermissionsResponse response =
1974       protocol.getUserPermissions(null, request);
1975     List<UserPermission> perms = new ArrayList<UserPermission>();
1976     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
1977       perms.add(ProtobufUtil.toUserPermission(perm));
1978     }
1979     return perms;
1980   }
1981 
1982   /**
1983    * Convert a protobuf UserTablePermissions to a
1984    * ListMultimap<String, TablePermission> where key is username.
1985    *
1986    * @param proto the protobuf UserPermission
1987    * @return the converted UserPermission
1988    */
1989   public static ListMultimap<String, TablePermission> toUserTablePermissions(
1990       AccessControlProtos.UsersAndPermissions proto) {
1991     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
1992     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
1993 
1994     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
1995       userPerm = proto.getUserPermissions(i);
1996       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
1997         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
1998         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
1999       }
2000     }
2001 
2002     return perms;
2003   }
2004 
2005   /**
2006    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2007    *
2008    * @param token the Token instance to copy
2009    * @return the protobuf Token message
2010    */
2011   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2012     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2013     builder.setIdentifier(ByteString.copyFrom(token.getIdentifier()));
2014     builder.setPassword(ByteString.copyFrom(token.getPassword()));
2015     if (token.getService() != null) {
2016       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2017     }
2018     return builder.build();
2019   }
2020 
2021   /**
2022    * Converts a protobuf Token message back into a Token instance.
2023    *
2024    * @param proto the protobuf Token message
2025    * @return the Token instance
2026    */
2027   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2028     return new Token<AuthenticationTokenIdentifier>(
2029         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2030         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2031         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2032         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2033   }
2034 
2035   /**
2036    * Find the HRegion encoded name based on a region specifier
2037    *
2038    * @param regionSpecifier the region specifier
2039    * @return the corresponding region's encoded name
2040    * @throws DoNotRetryIOException if the specifier type is unsupported
2041    */
2042   public static String getRegionEncodedName(
2043       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2044     byte[] value = regionSpecifier.getValue().toByteArray();
2045     RegionSpecifierType type = regionSpecifier.getType();
2046     switch (type) {
2047       case REGION_NAME:
2048         return HRegionInfo.encodeRegionName(value);
2049       case ENCODED_REGION_NAME:
2050         return Bytes.toString(value);
2051       default:
2052         throw new DoNotRetryIOException(
2053           "Unsupported region specifier type: " + type);
2054     }
2055   }
2056 
2057   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2058     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2059     try {
2060       builder.mergeFrom(bytes);
2061     } catch (InvalidProtocolBufferException e) {
2062       //Ignored there are just no key values to add.
2063     }
2064     MapReduceProtos.ScanMetrics pScanMetrics = builder.build();
2065     ScanMetrics scanMetrics = new ScanMetrics();
2066     for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2067       if (pair.hasName() && pair.hasValue()) {
2068         scanMetrics.setCounter(pair.getName(), pair.getValue());
2069       }
2070     }
2071     return scanMetrics;
2072   }
2073 
2074   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2075     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2076     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2077     for (Entry<String, Long> e : metrics.entrySet()) {
2078       HBaseProtos.NameInt64Pair nameInt64Pair =
2079           HBaseProtos.NameInt64Pair.newBuilder()
2080               .setName(e.getKey())
2081               .setValue(e.getValue())
2082               .build();
2083       builder.addMetrics(nameInt64Pair);
2084     }
2085     return builder.build();
2086   }
2087 
2088   /**
2089    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2090    * This method will <strong>always</strong> throw an exception.
2091    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2092    */
2093   public static void toIOException(ServiceException se) throws IOException {
2094     if (se == null) {
2095       throw new NullPointerException("Null service exception passed!");
2096     }
2097 
2098     Throwable cause = se.getCause();
2099     if (cause != null && cause instanceof IOException) {
2100       throw (IOException)cause;
2101     }
2102     throw new IOException(se);
2103   }
2104 
2105   public static CellProtos.Cell toCell(final Cell kv) {
2106     // Doing this is going to kill us if we do it for all data passed.
2107     // St.Ack 20121205
2108     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2109     kvbuilder.setRow(ByteString.copyFrom(kv.getRowArray(), kv.getRowOffset(),
2110       kv.getRowLength()));
2111     kvbuilder.setFamily(ByteString.copyFrom(kv.getFamilyArray(),
2112       kv.getFamilyOffset(), kv.getFamilyLength()));
2113     kvbuilder.setQualifier(ByteString.copyFrom(kv.getQualifierArray(),
2114       kv.getQualifierOffset(), kv.getQualifierLength()));
2115     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2116     kvbuilder.setTimestamp(kv.getTimestamp());
2117     kvbuilder.setValue(ByteString.copyFrom(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
2118     return kvbuilder.build();
2119   }
2120 
2121   public static Cell toCell(final CellProtos.Cell cell) {
2122     // Doing this is going to kill us if we do it for all data passed.
2123     // St.Ack 20121205
2124     return CellUtil.createCell(cell.getRow().toByteArray(),
2125       cell.getFamily().toByteArray(),
2126       cell.getQualifier().toByteArray(),
2127       cell.getTimestamp(),
2128       (byte)cell.getCellType().getNumber(),
2129       cell.getValue().toByteArray());
2130   }
2131 
2132   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2133     HBaseProtos.NamespaceDescriptor.Builder b =
2134         HBaseProtos.NamespaceDescriptor.newBuilder()
2135             .setName(ByteString.copyFromUtf8(ns.getName()));
2136     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2137       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2138           .setName(entry.getKey())
2139           .setValue(entry.getValue()));
2140     }
2141     return b.build();
2142   }
2143 
2144   public static NamespaceDescriptor toNamespaceDescriptor(
2145       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2146     NamespaceDescriptor.Builder b =
2147       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2148     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2149       b.addConfiguration(prop.getName(), prop.getValue());
2150     }
2151     return b.build();
2152   }
2153 
2154   /**
2155    * Get an instance of the argument type declared in a class's signature. The
2156    * argument type is assumed to be a PB Message subclass, and the instance is
2157    * created using parseFrom method on the passed ByteString.
2158    * @param runtimeClass the runtime type of the class
2159    * @param position the position of the argument in the class declaration
2160    * @param b the ByteString which should be parsed to get the instance created
2161    * @return the instance
2162    * @throws IOException
2163    */
2164   @SuppressWarnings("unchecked")
2165   public static <T extends Message>
2166   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2167       throws IOException {
2168     Type type = runtimeClass.getGenericSuperclass();
2169     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2170     Class<T> classType = (Class<T>)argType;
2171     T inst;
2172     try {
2173       Method m = classType.getMethod("parseFrom", ByteString.class);
2174       inst = (T)m.invoke(null, b);
2175       return inst;
2176     } catch (SecurityException e) {
2177       throw new IOException(e);
2178     } catch (NoSuchMethodException e) {
2179       throw new IOException(e);
2180     } catch (IllegalArgumentException e) {
2181       throw new IOException(e);
2182     } catch (InvocationTargetException e) {
2183       throw new IOException(e);
2184     } catch (IllegalAccessException e) {
2185       throw new IOException(e);
2186     }
2187   }
2188 
2189   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2190       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2191     // compaction descriptor contains relative paths.
2192     // input / output paths are relative to the store dir
2193     // store dir is relative to region dir
2194     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2195         .setTableName(ByteString.copyFrom(info.getTableName().getName()))
2196         .setEncodedRegionName(ByteString.copyFrom(info.getEncodedNameAsBytes()))
2197         .setFamilyName(ByteString.copyFrom(family))
2198         .setStoreHomeDir(storeDir.getName()); //make relative
2199     for (Path inputPath : inputPaths) {
2200       builder.addCompactionInput(inputPath.getName()); //relative path
2201     }
2202     for (Path outputPath : outputPaths) {
2203       builder.addCompactionOutput(outputPath.getName());
2204     }
2205     return builder.build();
2206   }
2207 
2208   /**
2209    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2210    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2211    * logs. Use judiciously.
2212    * @param m
2213    * @return toString of passed <code>m</code>
2214    */
2215   public static String getShortTextFormat(Message m) {
2216     if (m == null) return "null";
2217     if (m instanceof ScanRequest) {
2218       // This should be small and safe to output.  No data.
2219       return TextFormat.shortDebugString(m);
2220     } else if (m instanceof RegionServerReportRequest) {
2221       // Print a short message only, just the servername and the requests, not the full load.
2222       RegionServerReportRequest r = (RegionServerReportRequest)m;
2223       return "server " + TextFormat.shortDebugString(r.getServer()) +
2224         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2225     } else if (m instanceof RegionServerStartupRequest) {
2226       // Should be small enough.
2227       return TextFormat.shortDebugString(m);
2228     } else if (m instanceof MutationProto) {
2229       return toShortString((MutationProto)m);
2230     } else if (m instanceof GetRequest) {
2231       GetRequest r = (GetRequest) m;
2232       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2233           ", row=" + getStringForByteString(r.getGet().getRow());
2234     } else if (m instanceof ClientProtos.MultiRequest) {
2235       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2236       ClientProtos.MultiAction action = r.getActionList().get(0);
2237       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2238           ", for " + r.getActionCount() +
2239           " actions and 1st row key=" + getStringForByteString(action.hasMutation() ?
2240           action.getMutation().getRow() : action.getGet().getRow());
2241     } else if (m instanceof ClientProtos.MutateRequest) {
2242       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2243       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2244           ", row=" + getStringForByteString(r.getMutation().getRow());
2245     }
2246     return "TODO: " + m.getClass().toString();
2247   }
2248 
2249   private static String getStringForByteString(ByteString bs) {
2250     return Bytes.toStringBinary(bs.toByteArray());
2251   }
2252 
2253   /**
2254    * Print out some subset of a MutationProto rather than all of it and its data
2255    * @param proto Protobuf to print out
2256    * @return Short String of mutation proto
2257    */
2258   static String toShortString(final MutationProto proto) {
2259     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2260         ", type=" + proto.getMutateType().toString();
2261   }
2262 
2263   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2264     return TableName.valueOf(tableNamePB.getNamespace().toByteArray(),
2265         tableNamePB.getQualifier().toByteArray());
2266   }
2267 
2268   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2269     return HBaseProtos.TableName.newBuilder()
2270         .setNamespace(ByteString.copyFrom(tableName.getNamespace()))
2271         .setQualifier(ByteString.copyFrom(tableName.getQualifier())).build();
2272   }
2273 
2274   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2275     if (tableNamesList == null) {
2276       return new TableName[0];
2277     }
2278     TableName[] tableNames = new TableName[tableNamesList.size()];
2279     for (int i = 0; i < tableNamesList.size(); i++) {
2280       tableNames[i] = toTableName(tableNamesList.get(i));
2281     }
2282     return tableNames;
2283   }
2284 
2285 }