1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.rest.client;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Set;
29 import java.util.TreeMap;
30
31 import com.google.protobuf.Service;
32 import com.google.protobuf.ServiceException;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35
36 import org.apache.hadoop.hbase.client.*;
37 import org.apache.hadoop.hbase.client.coprocessor.Batch;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.util.StringUtils;
40
41 import org.apache.hadoop.classification.InterfaceAudience;
42 import org.apache.hadoop.classification.InterfaceStability;
43 import org.apache.hadoop.conf.Configuration;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HConstants;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.KeyValue;
49 import org.apache.hadoop.hbase.KeyValueUtil;
50 import org.apache.hadoop.hbase.client.RowMutations;
51 import org.apache.hadoop.hbase.client.Delete;
52 import org.apache.hadoop.hbase.client.Get;
53 import org.apache.hadoop.hbase.client.HTableInterface;
54 import org.apache.hadoop.hbase.client.Increment;
55 import org.apache.hadoop.hbase.client.Put;
56 import org.apache.hadoop.hbase.client.Row;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.client.ResultScanner;
59 import org.apache.hadoop.hbase.client.Scan;
60 import org.apache.hadoop.hbase.io.TimeRange;
61 import org.apache.hadoop.hbase.rest.Constants;
62 import org.apache.hadoop.hbase.rest.model.CellModel;
63 import org.apache.hadoop.hbase.rest.model.CellSetModel;
64 import org.apache.hadoop.hbase.rest.model.RowModel;
65 import org.apache.hadoop.hbase.rest.model.ScannerModel;
66 import org.apache.hadoop.hbase.rest.model.TableSchemaModel;
67 import org.apache.hadoop.hbase.util.Bytes;
68
69
70
71
72 @InterfaceAudience.Public
73 @InterfaceStability.Stable
74 public class RemoteHTable implements HTableInterface {
75
76 private static final Log LOG = LogFactory.getLog(RemoteHTable.class);
77
78 final Client client;
79 final Configuration conf;
80 final byte[] name;
81 final int maxRetries;
82 final long sleepTime;
83
84 @SuppressWarnings("rawtypes")
85 protected String buildRowSpec(final byte[] row, final Map familyMap,
86 final long startTime, final long endTime, final int maxVersions) {
87 StringBuffer sb = new StringBuffer();
88 sb.append('/');
89 sb.append(Bytes.toStringBinary(name));
90 sb.append('/');
91 sb.append(Bytes.toStringBinary(row));
92 Set families = familyMap.entrySet();
93 if (families != null) {
94 Iterator i = familyMap.entrySet().iterator();
95 if (i.hasNext()) {
96 sb.append('/');
97 }
98 while (i.hasNext()) {
99 Map.Entry e = (Map.Entry)i.next();
100 Collection quals = (Collection)e.getValue();
101 if (quals != null && !quals.isEmpty()) {
102 Iterator ii = quals.iterator();
103 while (ii.hasNext()) {
104 sb.append(Bytes.toStringBinary((byte[])e.getKey()));
105 sb.append(':');
106 Object o = ii.next();
107
108 if (o instanceof byte[]) {
109 sb.append(Bytes.toStringBinary((byte[])o));
110 } else if (o instanceof KeyValue) {
111 sb.append(Bytes.toStringBinary(((KeyValue)o).getQualifier()));
112 } else {
113 throw new RuntimeException("object type not handled");
114 }
115 if (ii.hasNext()) {
116 sb.append(',');
117 }
118 }
119 } else {
120 sb.append(Bytes.toStringBinary((byte[])e.getKey()));
121 sb.append(':');
122 }
123 if (i.hasNext()) {
124 sb.append(',');
125 }
126 }
127 }
128 if (startTime != 0 && endTime != Long.MAX_VALUE) {
129 sb.append('/');
130 sb.append(startTime);
131 if (startTime != endTime) {
132 sb.append(',');
133 sb.append(endTime);
134 }
135 } else if (endTime != Long.MAX_VALUE) {
136 sb.append('/');
137 sb.append(endTime);
138 }
139 if (maxVersions > 1) {
140 sb.append("?v=");
141 sb.append(maxVersions);
142 }
143 return sb.toString();
144 }
145
146 protected String buildMultiRowSpec(final byte[][] rows, int maxVersions) {
147 StringBuilder sb = new StringBuilder();
148 sb.append('/');
149 sb.append(Bytes.toStringBinary(name));
150 sb.append("/multiget/");
151 if (rows == null || rows.length == 0) {
152 return sb.toString();
153 }
154 sb.append("?");
155 for(int i=0; i<rows.length; i++) {
156 byte[] rk = rows[i];
157 if (i != 0) {
158 sb.append('&');
159 }
160 sb.append("row=");
161 sb.append(Bytes.toStringBinary(rk));
162 }
163 sb.append("&v=");
164 sb.append(maxVersions);
165
166 return sb.toString();
167 }
168
169 protected Result[] buildResultFromModel(final CellSetModel model) {
170 List<Result> results = new ArrayList<Result>();
171 for (RowModel row: model.getRows()) {
172 List<KeyValue> kvs = new ArrayList<KeyValue>();
173 for (CellModel cell: row.getCells()) {
174 byte[][] split = KeyValue.parseColumn(cell.getColumn());
175 byte[] column = split[0];
176 byte[] qualifier = split.length > 1 ? split[1] : null;
177 kvs.add(new KeyValue(row.getKey(), column, qualifier,
178 cell.getTimestamp(), cell.getValue()));
179 }
180 results.add(new Result(kvs));
181 }
182 return results.toArray(new Result[results.size()]);
183 }
184
185 protected CellSetModel buildModelFromPut(Put put) {
186 RowModel row = new RowModel(put.getRow());
187 long ts = put.getTimeStamp();
188 for (List<? extends Cell> cells: put.getFamilyMap().values()) {
189 for (Cell cell: cells) {
190 KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
191 row.addCell(new CellModel(kv.getFamily(), kv.getQualifier(),
192 ts != HConstants.LATEST_TIMESTAMP ? ts : kv.getTimestamp(),
193 kv.getValue()));
194 }
195 }
196 CellSetModel model = new CellSetModel();
197 model.addRow(row);
198 return model;
199 }
200
201
202
203
204
205
206 public RemoteHTable(Client client, String name) {
207 this(client, HBaseConfiguration.create(), Bytes.toBytes(name));
208 }
209
210
211
212
213
214
215
216 public RemoteHTable(Client client, Configuration conf, String name) {
217 this(client, conf, Bytes.toBytes(name));
218 }
219
220
221
222
223
224
225
226 public RemoteHTable(Client client, Configuration conf, byte[] name) {
227 this.client = client;
228 this.conf = conf;
229 this.name = name;
230 this.maxRetries = conf.getInt("hbase.rest.client.max.retries", 10);
231 this.sleepTime = conf.getLong("hbase.rest.client.sleep", 1000);
232 }
233
234 public byte[] getTableName() {
235 return name.clone();
236 }
237
238 public Configuration getConfiguration() {
239 return conf;
240 }
241
242 public HTableDescriptor getTableDescriptor() throws IOException {
243 StringBuilder sb = new StringBuilder();
244 sb.append('/');
245 sb.append(Bytes.toStringBinary(name));
246 sb.append('/');
247 sb.append("schema");
248 for (int i = 0; i < maxRetries; i++) {
249 Response response = client.get(sb.toString(), Constants.MIMETYPE_PROTOBUF);
250 int code = response.getCode();
251 switch (code) {
252 case 200:
253 TableSchemaModel schema = new TableSchemaModel();
254 schema.getObjectFromMessage(response.getBody());
255 return schema.getTableDescriptor();
256 case 509:
257 try {
258 Thread.sleep(sleepTime);
259 } catch (InterruptedException e) { }
260 break;
261 default:
262 throw new IOException("schema request returned " + code);
263 }
264 }
265 throw new IOException("schema request timed out");
266 }
267
268 public void close() throws IOException {
269 client.shutdown();
270 }
271
272 public Result get(Get get) throws IOException {
273 TimeRange range = get.getTimeRange();
274 String spec = buildRowSpec(get.getRow(), get.getFamilyMap(),
275 range.getMin(), range.getMax(), get.getMaxVersions());
276 if (get.getFilter() != null) {
277 LOG.warn("filters not supported on gets");
278 }
279 Result[] results = getResults(spec);
280 if (results.length > 0) {
281 if (results.length > 1) {
282 LOG.warn("too many results for get (" + results.length + ")");
283 }
284 return results[0];
285 } else {
286 return new Result();
287 }
288 }
289
290 public Result[] get(List<Get> gets) throws IOException {
291 byte[][] rows = new byte[gets.size()][];
292 int maxVersions = 1;
293 int count = 0;
294
295 for(Get g:gets) {
296
297 if ( count == 0 ) {
298 maxVersions = g.getMaxVersions();
299 } else if (g.getMaxVersions() != maxVersions) {
300 LOG.warn("MaxVersions on Gets do not match, using the first in the list ("+maxVersions+")");
301 }
302
303 if (g.getFilter() != null) {
304 LOG.warn("filters not supported on gets");
305 }
306
307 rows[count] = g.getRow();
308 count ++;
309 }
310
311 String spec = buildMultiRowSpec(rows, maxVersions);
312
313 return getResults(spec);
314 }
315
316 private Result[] getResults(String spec) throws IOException {
317 for (int i = 0; i < maxRetries; i++) {
318 Response response = client.get(spec, Constants.MIMETYPE_PROTOBUF);
319 int code = response.getCode();
320 switch (code) {
321 case 200:
322 CellSetModel model = new CellSetModel();
323 model.getObjectFromMessage(response.getBody());
324 Result[] results = buildResultFromModel(model);
325 if ( results.length > 0) {
326 return results;
327 }
328
329 case 404:
330 return new Result[0];
331
332 case 509:
333 try {
334 Thread.sleep(sleepTime);
335 } catch (InterruptedException e) { }
336 break;
337 default:
338 throw new IOException("get request returned " + code);
339 }
340 }
341 throw new IOException("get request timed out");
342 }
343
344 public boolean exists(Get get) throws IOException {
345 LOG.warn("exists() is really get(), just use get()");
346 Result result = get(get);
347 return (result != null && !(result.isEmpty()));
348 }
349
350
351
352
353
354 public Boolean[] exists(List<Get> gets) throws IOException {
355 LOG.warn("exists(List<Get>) is really list of get() calls, just use get()");
356 Boolean[] results = new Boolean[gets.size()];
357 for (int i = 0; i < results.length; i++) {
358 results[i] = exists(gets.get(i));
359 }
360 return results;
361 }
362
363 public void put(Put put) throws IOException {
364 CellSetModel model = buildModelFromPut(put);
365 StringBuilder sb = new StringBuilder();
366 sb.append('/');
367 sb.append(Bytes.toStringBinary(name));
368 sb.append('/');
369 sb.append(Bytes.toStringBinary(put.getRow()));
370 for (int i = 0; i < maxRetries; i++) {
371 Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF,
372 model.createProtobufOutput());
373 int code = response.getCode();
374 switch (code) {
375 case 200:
376 return;
377 case 509:
378 try {
379 Thread.sleep(sleepTime);
380 } catch (InterruptedException e) { }
381 break;
382 default:
383 throw new IOException("put request failed with " + code);
384 }
385 }
386 throw new IOException("put request timed out");
387 }
388
389 public void put(List<Put> puts) throws IOException {
390
391
392
393
394 TreeMap<byte[],List<Cell>> map =
395 new TreeMap<byte[],List<Cell>>(Bytes.BYTES_COMPARATOR);
396 for (Put put: puts) {
397 byte[] row = put.getRow();
398 List<Cell> cells = map.get(row);
399 if (cells == null) {
400 cells = new ArrayList<Cell>();
401 map.put(row, cells);
402 }
403 for (List<? extends Cell> l: put.getFamilyMap().values()) {
404 cells.addAll(l);
405 }
406 }
407
408
409 CellSetModel model = new CellSetModel();
410 for (Map.Entry<byte[], List<Cell>> e: map.entrySet()) {
411 RowModel row = new RowModel(e.getKey());
412 for (Cell cell: e.getValue()) {
413 KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
414 row.addCell(new CellModel(kv));
415 }
416 model.addRow(row);
417 }
418
419
420 StringBuilder sb = new StringBuilder();
421 sb.append('/');
422 sb.append(Bytes.toStringBinary(name));
423 sb.append("/$multiput");
424 for (int i = 0; i < maxRetries; i++) {
425 Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF,
426 model.createProtobufOutput());
427 int code = response.getCode();
428 switch (code) {
429 case 200:
430 return;
431 case 509:
432 try {
433 Thread.sleep(sleepTime);
434 } catch (InterruptedException e) { }
435 break;
436 default:
437 throw new IOException("multiput request failed with " + code);
438 }
439 }
440 throw new IOException("multiput request timed out");
441 }
442
443 public void delete(Delete delete) throws IOException {
444 String spec = buildRowSpec(delete.getRow(), delete.getFamilyMap(),
445 delete.getTimeStamp(), delete.getTimeStamp(), 1);
446 for (int i = 0; i < maxRetries; i++) {
447 Response response = client.delete(spec);
448 int code = response.getCode();
449 switch (code) {
450 case 200:
451 return;
452 case 509:
453 try {
454 Thread.sleep(sleepTime);
455 } catch (InterruptedException e) { }
456 break;
457 default:
458 throw new IOException("delete request failed with " + code);
459 }
460 }
461 throw new IOException("delete request timed out");
462 }
463
464 public void delete(List<Delete> deletes) throws IOException {
465 for (Delete delete: deletes) {
466 delete(delete);
467 }
468 }
469
470 public void flushCommits() throws IOException {
471
472 }
473
474 class Scanner implements ResultScanner {
475
476 String uri;
477
478 public Scanner(Scan scan) throws IOException {
479 ScannerModel model;
480 try {
481 model = ScannerModel.fromScan(scan);
482 } catch (Exception e) {
483 throw new IOException(e);
484 }
485 StringBuffer sb = new StringBuffer();
486 sb.append('/');
487 sb.append(Bytes.toStringBinary(name));
488 sb.append('/');
489 sb.append("scanner");
490 for (int i = 0; i < maxRetries; i++) {
491 Response response = client.post(sb.toString(),
492 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
493 int code = response.getCode();
494 switch (code) {
495 case 201:
496 uri = response.getLocation();
497 return;
498 case 509:
499 try {
500 Thread.sleep(sleepTime);
501 } catch (InterruptedException e) { }
502 break;
503 default:
504 throw new IOException("scan request failed with " + code);
505 }
506 }
507 throw new IOException("scan request timed out");
508 }
509
510 @Override
511 public Result[] next(int nbRows) throws IOException {
512 StringBuilder sb = new StringBuilder(uri);
513 sb.append("?n=");
514 sb.append(nbRows);
515 for (int i = 0; i < maxRetries; i++) {
516 Response response = client.get(sb.toString(),
517 Constants.MIMETYPE_PROTOBUF);
518 int code = response.getCode();
519 switch (code) {
520 case 200:
521 CellSetModel model = new CellSetModel();
522 model.getObjectFromMessage(response.getBody());
523 return buildResultFromModel(model);
524 case 204:
525 case 206:
526 return null;
527 case 509:
528 try {
529 Thread.sleep(sleepTime);
530 } catch (InterruptedException e) { }
531 break;
532 default:
533 throw new IOException("scanner.next request failed with " + code);
534 }
535 }
536 throw new IOException("scanner.next request timed out");
537 }
538
539 @Override
540 public Result next() throws IOException {
541 Result[] results = next(1);
542 if (results == null || results.length < 1) {
543 return null;
544 }
545 return results[0];
546 }
547
548 class Iter implements Iterator<Result> {
549
550 Result cache;
551
552 public Iter() {
553 try {
554 cache = Scanner.this.next();
555 } catch (IOException e) {
556 LOG.warn(StringUtils.stringifyException(e));
557 }
558 }
559
560 @Override
561 public boolean hasNext() {
562 return cache != null;
563 }
564
565 @Override
566 public Result next() {
567 Result result = cache;
568 try {
569 cache = Scanner.this.next();
570 } catch (IOException e) {
571 LOG.warn(StringUtils.stringifyException(e));
572 cache = null;
573 }
574 return result;
575 }
576
577 @Override
578 public void remove() {
579 throw new RuntimeException("remove() not supported");
580 }
581
582 }
583
584 @Override
585 public Iterator<Result> iterator() {
586 return new Iter();
587 }
588
589 @Override
590 public void close() {
591 try {
592 client.delete(uri);
593 } catch (IOException e) {
594 LOG.warn(StringUtils.stringifyException(e));
595 }
596 }
597
598 }
599
600 public ResultScanner getScanner(Scan scan) throws IOException {
601 return new Scanner(scan);
602 }
603
604 public ResultScanner getScanner(byte[] family) throws IOException {
605 Scan scan = new Scan();
606 scan.addFamily(family);
607 return new Scanner(scan);
608 }
609
610 public ResultScanner getScanner(byte[] family, byte[] qualifier)
611 throws IOException {
612 Scan scan = new Scan();
613 scan.addColumn(family, qualifier);
614 return new Scanner(scan);
615 }
616
617 public boolean isAutoFlush() {
618 return true;
619 }
620
621 public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
622 throw new IOException("getRowOrBefore not supported");
623 }
624
625 public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
626 byte[] value, Put put) throws IOException {
627
628 put.add(new KeyValue(row, family, qualifier, value));
629
630 CellSetModel model = buildModelFromPut(put);
631 StringBuilder sb = new StringBuilder();
632 sb.append('/');
633 sb.append(Bytes.toStringBinary(name));
634 sb.append('/');
635 sb.append(Bytes.toStringBinary(put.getRow()));
636 sb.append("?check=put");
637
638 for (int i = 0; i < maxRetries; i++) {
639 Response response = client.put(sb.toString(),
640 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
641 int code = response.getCode();
642 switch (code) {
643 case 200:
644 return true;
645 case 304:
646 return false;
647 case 509:
648 try {
649 Thread.sleep(sleepTime);
650 } catch (final InterruptedException e) {
651 }
652 break;
653 default:
654 throw new IOException("checkAndPut request failed with " + code);
655 }
656 }
657 throw new IOException("checkAndPut request timed out");
658 }
659
660 public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
661 byte[] value, Delete delete) throws IOException {
662 Put put = new Put(row);
663
664 put.add(new KeyValue(row, family, qualifier, value));
665 CellSetModel model = buildModelFromPut(put);
666 StringBuilder sb = new StringBuilder();
667 sb.append('/');
668 sb.append(Bytes.toStringBinary(name));
669 sb.append('/');
670 sb.append(Bytes.toStringBinary(row));
671 sb.append("?check=delete");
672
673 for (int i = 0; i < maxRetries; i++) {
674 Response response = client.put(sb.toString(),
675 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
676 int code = response.getCode();
677 switch (code) {
678 case 200:
679 return true;
680 case 304:
681 return false;
682 case 509:
683 try {
684 Thread.sleep(sleepTime);
685 } catch (final InterruptedException e) {
686 }
687 break;
688 default:
689 throw new IOException("checkAndDelete request failed with " + code);
690 }
691 }
692 throw new IOException("checkAndDelete request timed out");
693 }
694
695 public Result increment(Increment increment) throws IOException {
696 throw new IOException("Increment not supported");
697 }
698
699 public Result append(Append append) throws IOException {
700 throw new IOException("Append not supported");
701 }
702
703 public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
704 long amount) throws IOException {
705 throw new IOException("incrementColumnValue not supported");
706 }
707
708 public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
709 long amount, boolean writeToWAL) throws IOException {
710 throw new IOException("incrementColumnValue not supported");
711 }
712
713 @Override
714 public void batch(List<? extends Row> actions, Object[] results) throws IOException {
715 throw new IOException("batch not supported");
716 }
717
718 @Override
719 public Object[] batch(List<? extends Row> actions) throws IOException {
720 throw new IOException("batch not supported");
721 }
722
723 @Override
724 public <R> void batchCallback(List<? extends Row> actions, Object[] results,
725 Batch.Callback<R> callback) throws IOException, InterruptedException {
726 throw new IOException("batchCallback not supported");
727 }
728
729 @Override
730 public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback)
731 throws IOException, InterruptedException {
732 throw new IOException("batchCallback not supported");
733 }
734
735 @Override
736 public CoprocessorRpcChannel coprocessorService(byte[] row) {
737 throw new UnsupportedOperationException("coprocessorService not implemented");
738 }
739
740 @Override
741 public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service,
742 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable)
743 throws ServiceException, Throwable {
744 throw new UnsupportedOperationException("coprocessorService not implemented");
745 }
746
747 @Override
748 public <T extends Service, R> void coprocessorService(Class<T> service,
749 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable, Batch.Callback<R> callback)
750 throws ServiceException, Throwable {
751 throw new UnsupportedOperationException("coprocessorService not implemented");
752 }
753
754 @Override
755 public void mutateRow(RowMutations rm) throws IOException {
756 throw new IOException("atomicMutation not supported");
757 }
758
759 @Override
760 public void setAutoFlush(boolean autoFlush) {
761 throw new UnsupportedOperationException("setAutoFlush not implemented");
762 }
763
764 @Override
765 public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
766 throw new UnsupportedOperationException("setAutoFlush not implemented");
767 }
768
769 @Override
770 public long getWriteBufferSize() {
771 throw new UnsupportedOperationException("getWriteBufferSize not implemented");
772 }
773
774 @Override
775 public void setWriteBufferSize(long writeBufferSize) throws IOException {
776 throw new IOException("setWriteBufferSize not supported");
777 }
778 }