View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import javax.annotation.Nullable;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.CellScanner;
30  import org.apache.hadoop.hbase.DoNotRetryIOException;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.ServerName;
33  import org.apache.hadoop.hbase.client.Result;
34  import org.apache.hadoop.hbase.ipc.ServerRpcController;
35  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
36  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
42  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
48  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
50  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
51  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
52  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
53  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
54  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
55  import org.apache.hadoop.hbase.security.access.UserPermission;
56  import org.apache.hadoop.hbase.util.Pair;
57  import org.apache.hadoop.util.StringUtils;
58  
59  import com.google.protobuf.ByteString;
60  import com.google.protobuf.RpcController;
61  
62  /**
63   * Helper utility to build protocol buffer responses,
64   * or retrieve data from protocol buffer responses.
65   */
66  @InterfaceAudience.Private
67  public final class ResponseConverter {
68    public static final Log LOG = LogFactory.getLog(ResponseConverter.class);
69  
70    private ResponseConverter() {
71    }
72  
73  // Start utilities for Client
74  
75    /**
76     * Get the results from a protocol buffer MultiResponse
77     *
78     * @param request the protocol buffer MultiResponse to convert
79     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
80     * @return the results that were in the MultiResponse (a Result or an Exception).
81     * @throws IOException
82     */
83    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
84        final MultiResponse response, final CellScanner cells)
85    throws IOException {
86      int requestRegionActionCount = request.getRegionActionCount();
87      int responseRegionActionResultCount = response.getRegionActionResultCount();
88      if (requestRegionActionCount != responseRegionActionResultCount) {
89        throw new IllegalStateException("Request mutation count=" + requestRegionActionCount +
90            " does not match response mutation result count=" + responseRegionActionResultCount);
91      }
92  
93      org.apache.hadoop.hbase.client.MultiResponse results =
94        new org.apache.hadoop.hbase.client.MultiResponse();
95  
96      for (int i = 0; i < responseRegionActionResultCount; i++) {
97        RegionAction actions = request.getRegionAction(i);
98        RegionActionResult actionResult = response.getRegionActionResult(i);
99        HBaseProtos.RegionSpecifier rs = actions.getRegion();
100       if (rs.hasType() &&
101           (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
102         throw new IllegalArgumentException(
103             "We support only encoded types for protobuf multi response.");
104       }
105       byte[] regionName = rs.getValue().toByteArray();
106 
107       if (actionResult.hasException()){
108         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
109         results.addException(regionName, regionException);
110         continue;
111       }
112 
113       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
114         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
115             ", actionResult.getResultOrExceptionCount=" +
116             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
117       }
118 
119       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
120         Object responseValue;
121         if (roe.hasException()) {
122           responseValue = ProtobufUtil.toException(roe.getException());
123         } else if (roe.hasResult()) {
124           responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
125         } else if (roe.hasServiceResult()) {
126           responseValue = roe.getServiceResult();
127         } else{
128           // Sometimes, the response is just "it was processed". Generally, this occurs for things
129           // like mutateRows where either we get back 'processed' (or not) and optionally some
130           // statistics about the regions we touched.
131           responseValue = response.getProcessed() ?
132                           ProtobufUtil.EMPTY_RESULT_EXISTS_TRUE :
133                           ProtobufUtil.EMPTY_RESULT_EXISTS_FALSE;
134         }
135         results.add(regionName, roe.getIndex(), responseValue);
136       }
137     }
138 
139     if (response.hasRegionStatistics()) {
140       ClientProtos.MultiRegionLoadStats stats = response.getRegionStatistics();
141       for (int i = 0; i < stats.getRegionCount(); i++) {
142         results.addStatistic(stats.getRegion(i).getValue().toByteArray(), stats.getStat(i));
143       }
144     }
145 
146     return results;
147   }
148 
149   /**
150    * Wrap a throwable to an action result.
151    *
152    * @param t
153    * @return an action result builder
154    */
155   public static ResultOrException.Builder buildActionResult(final Throwable t) {
156     ResultOrException.Builder builder = ResultOrException.newBuilder();
157     if (t != null) builder.setException(buildException(t));
158     return builder;
159   }
160 
161   /**
162    * Wrap a throwable to an action result.
163    *
164    * @param r
165    * @return an action result builder
166    */
167   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r) {
168     ResultOrException.Builder builder = ResultOrException.newBuilder();
169     if (r != null) builder.setResult(r);
170     return builder;
171   }
172 
173   /**
174    * @param t
175    * @return NameValuePair of the exception name to stringified version os exception.
176    */
177   public static NameBytesPair buildException(final Throwable t) {
178     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
179     parameterBuilder.setName(t.getClass().getName());
180     parameterBuilder.setValue(
181       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
182     return parameterBuilder.build();
183   }
184 
185   /**
186    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
187    */
188   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
189       final List<UserPermission> permissions) {
190     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
191     for (UserPermission perm : permissions) {
192       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
193     }
194     return builder.build();
195   }
196 
197 // End utilities for Client
198 // Start utilities for Admin
199 
200   /**
201    * Get the list of regions to flush from a RollLogWriterResponse
202    *
203    * @param proto the RollLogWriterResponse
204    * @return the the list of regions to flush
205    */
206   public static byte[][] getRegions(final RollWALWriterResponse proto) {
207     if (proto == null || proto.getRegionToFlushCount() == 0) return null;
208     List<byte[]> regions = new ArrayList<byte[]>();
209     for (ByteString region: proto.getRegionToFlushList()) {
210       regions.add(region.toByteArray());
211     }
212     return (byte[][])regions.toArray();
213   }
214 
215   /**
216    * Get the list of region info from a GetOnlineRegionResponse
217    *
218    * @param proto the GetOnlineRegionResponse
219    * @return the list of region info
220    */
221   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
222     if (proto == null || proto.getRegionInfoCount() == 0) return null;
223     return ProtobufUtil.getRegionInfos(proto);
224   }
225 
226   /**
227    * Get the region opening state from a OpenRegionResponse
228    *
229    * @param proto the OpenRegionResponse
230    * @return the region opening state
231    */
232   public static RegionOpeningState getRegionOpeningState
233       (final OpenRegionResponse proto) {
234     if (proto == null || proto.getOpeningStateCount() != 1) return null;
235     return RegionOpeningState.valueOf(
236       proto.getOpeningState(0).name());
237   }
238 
239   /**
240    * Get a list of region opening state from a OpenRegionResponse
241    * 
242    * @param proto the OpenRegionResponse
243    * @return the list of region opening state
244    */
245   public static List<RegionOpeningState> getRegionOpeningStateList(
246       final OpenRegionResponse proto) {
247     if (proto == null) return null;
248     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
249     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
250       regionOpeningStates.add(RegionOpeningState.valueOf(
251           proto.getOpeningState(i).name()));
252     }
253     return regionOpeningStates;
254   }
255 
256   /**
257    * Check if the region is closed from a CloseRegionResponse
258    *
259    * @param proto the CloseRegionResponse
260    * @return the region close state
261    */
262   public static boolean isClosed
263       (final CloseRegionResponse proto) {
264     if (proto == null || !proto.hasClosed()) return false;
265     return proto.getClosed();
266   }
267 
268   /**
269    * A utility to build a GetServerInfoResponse.
270    *
271    * @param serverName
272    * @param webuiPort
273    * @return the response
274    */
275   public static GetServerInfoResponse buildGetServerInfoResponse(
276       final ServerName serverName, final int webuiPort) {
277     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
278     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
279     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
280     if (webuiPort >= 0) {
281       serverInfoBuilder.setWebuiPort(webuiPort);
282     }
283     builder.setServerInfo(serverInfoBuilder.build());
284     return builder.build();
285   }
286 
287   /**
288    * A utility to build a GetOnlineRegionResponse.
289    *
290    * @param regions
291    * @return the response
292    */
293   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
294       final List<HRegionInfo> regions) {
295     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
296     for (HRegionInfo region: regions) {
297       builder.addRegionInfo(HRegionInfo.convert(region));
298     }
299     return builder.build();
300   }
301 
302   /**
303    * Creates a response for the catalog scan request
304    * @return A RunCatalogScanResponse
305    */
306   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
307     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
308   }
309 
310   /**
311    * Creates a response for the catalog scan request
312    * @return A EnableCatalogJanitorResponse
313    */
314   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
315     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
316   }
317 
318 // End utilities for Admin
319 
320   /**
321    * Creates a response for the last flushed sequence Id request
322    * @return A GetLastFlushedSequenceIdResponse
323    */
324   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
325       long seqId) {
326     return GetLastFlushedSequenceIdResponse.newBuilder().setLastFlushedSequenceId(seqId).build();
327   }
328 
329   /**
330    * Stores an exception encountered during RPC invocation so it can be passed back
331    * through to the client.
332    * @param controller the controller instance provided by the client when calling the service
333    * @param ioe the exception encountered
334    */
335   public static void setControllerException(RpcController controller, IOException ioe) {
336     if (controller != null) {
337       if (controller instanceof ServerRpcController) {
338         ((ServerRpcController)controller).setFailedOn(ioe);
339       } else {
340         controller.setFailed(StringUtils.stringifyException(ioe));
341       }
342     }
343   }
344 
345   /**
346    * Retreivies exception stored during RPC invocation.
347    * @param controller the controller instance provided by the client when calling the service
348    * @return exception if any, or null; Will return DoNotRetryIOException for string represented
349    * failure causes in controller.
350    */
351   @Nullable
352   public static IOException getControllerException(RpcController controller) throws IOException {
353     if (controller != null && controller.failed()) {
354       if (controller instanceof ServerRpcController) {
355         return ((ServerRpcController)controller).getFailedOn();
356       } else {
357         return new DoNotRetryIOException(controller.errorText());
358       }
359     }
360     return null;
361   }
362 
363 
364   /**
365    * Create Results from the cells using the cells meta data. 
366    * @param cellScanner
367    * @param response
368    * @return results
369    */
370   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
371       throws IOException {
372     if (response == null) return null;
373     // If cellscanner, then the number of Results to return is the count of elements in the
374     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
375     int noOfResults = cellScanner != null?
376       response.getCellsPerResultCount(): response.getResultsCount();
377     Result[] results = new Result[noOfResults];
378     for (int i = 0; i < noOfResults; i++) {
379       if (cellScanner != null) {
380         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
381         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
382         int noOfCells = response.getCellsPerResult(i);
383         List<Cell> cells = new ArrayList<Cell>(noOfCells);
384         for (int j = 0; j < noOfCells; j++) {
385           try {
386             if (cellScanner.advance() == false) {
387               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
388               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
389               // same scanner will result in OutOfOrderScannerNextException
390               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
391                 + " results completely at client. Resetting the scanner to scan again.";
392               LOG.error(msg);
393               throw new DoNotRetryIOException(msg);
394             }
395           } catch (IOException ioe) {
396             // We are getting IOE while retrieving the cells for Results.
397             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
398             // same scanner will result in OutOfOrderScannerNextException
399             LOG.error("Exception while reading cells from result."
400               + "Resetting the scanner to scan again.", ioe);
401             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
402           }
403           cells.add(cellScanner.current());
404         }
405         results[i] = Result.create(cells);
406       } else {
407         // Result is pure pb.
408         results[i] = ProtobufUtil.toResult(response.getResults(i));
409       }
410     }
411     return results;
412   }
413 }