1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.rest.client;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.TreeMap;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34
35 import org.apache.hadoop.hbase.client.*;
36 import org.apache.hadoop.hbase.client.coprocessor.Batch;
37 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
38 import org.apache.hadoop.util.StringUtils;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.KeyValue;
45 import org.apache.hadoop.hbase.client.RowMutations;
46 import org.apache.hadoop.hbase.client.Delete;
47 import org.apache.hadoop.hbase.client.Get;
48 import org.apache.hadoop.hbase.client.HTableInterface;
49 import org.apache.hadoop.hbase.client.Increment;
50 import org.apache.hadoop.hbase.client.Put;
51 import org.apache.hadoop.hbase.client.Row;
52 import org.apache.hadoop.hbase.client.Result;
53 import org.apache.hadoop.hbase.client.ResultScanner;
54 import org.apache.hadoop.hbase.client.RowLock;
55 import org.apache.hadoop.hbase.client.Scan;
56 import org.apache.hadoop.hbase.io.TimeRange;
57 import org.apache.hadoop.hbase.rest.Constants;
58 import org.apache.hadoop.hbase.rest.model.CellModel;
59 import org.apache.hadoop.hbase.rest.model.CellSetModel;
60 import org.apache.hadoop.hbase.rest.model.RowModel;
61 import org.apache.hadoop.hbase.rest.model.ScannerModel;
62 import org.apache.hadoop.hbase.rest.model.TableSchemaModel;
63 import org.apache.hadoop.hbase.util.Bytes;
64
65
66
67
68 public class RemoteHTable implements HTableInterface {
69
70 private static final Log LOG = LogFactory.getLog(RemoteHTable.class);
71
72 final Client client;
73 final Configuration conf;
74 final byte[] name;
75 final int maxRetries;
76 final long sleepTime;
77
78 @SuppressWarnings("rawtypes")
79 protected String buildRowSpec(final byte[] row, final Map familyMap,
80 final long startTime, final long endTime, final int maxVersions) {
81 StringBuffer sb = new StringBuffer();
82 sb.append('/');
83 sb.append(Bytes.toStringBinary(name));
84 sb.append('/');
85 sb.append(Bytes.toStringBinary(row));
86 Set families = familyMap.entrySet();
87 if (families != null) {
88 Iterator i = familyMap.entrySet().iterator();
89 sb.append('/');
90 while (i.hasNext()) {
91 Map.Entry e = (Map.Entry)i.next();
92 Collection quals = (Collection)e.getValue();
93 if (quals != null && !quals.isEmpty()) {
94 Iterator ii = quals.iterator();
95 while (ii.hasNext()) {
96 sb.append(Bytes.toStringBinary((byte[])e.getKey()));
97 sb.append(':');
98 Object o = ii.next();
99
100 if (o instanceof byte[]) {
101 sb.append(Bytes.toStringBinary((byte[])o));
102 } else if (o instanceof KeyValue) {
103 sb.append(Bytes.toStringBinary(((KeyValue)o).getQualifier()));
104 } else {
105 throw new RuntimeException("object type not handled");
106 }
107 if (ii.hasNext()) {
108 sb.append(',');
109 }
110 }
111 } else {
112 sb.append(Bytes.toStringBinary((byte[])e.getKey()));
113 sb.append(':');
114 }
115 if (i.hasNext()) {
116 sb.append(',');
117 }
118 }
119 }
120 if (startTime != 0 && endTime != Long.MAX_VALUE) {
121 sb.append('/');
122 sb.append(startTime);
123 if (startTime != endTime) {
124 sb.append(',');
125 sb.append(endTime);
126 }
127 } else if (endTime != Long.MAX_VALUE) {
128 sb.append('/');
129 sb.append(endTime);
130 }
131 if (maxVersions > 1) {
132 sb.append("?v=");
133 sb.append(maxVersions);
134 }
135 return sb.toString();
136 }
137
138 protected String buildMultiRowSpec(final byte[][] rows, int maxVersions) {
139 StringBuilder sb = new StringBuilder();
140 sb.append('/');
141 sb.append(Bytes.toStringBinary(name));
142 sb.append("/multiget/");
143 if (rows == null || rows.length == 0) {
144 return sb.toString();
145 }
146 sb.append("?");
147 for(int i=0; i<rows.length; i++) {
148 byte[] rk = rows[i];
149 if (i != 0) {
150 sb.append('&');
151 }
152 sb.append("row=");
153 sb.append(Bytes.toStringBinary(rk));
154 }
155 sb.append("&v=");
156 sb.append(maxVersions);
157
158 return sb.toString();
159 }
160
161 protected Result[] buildResultFromModel(final CellSetModel model) {
162 List<Result> results = new ArrayList<Result>();
163 for (RowModel row: model.getRows()) {
164 List<KeyValue> kvs = new ArrayList<KeyValue>();
165 for (CellModel cell: row.getCells()) {
166 byte[][] split = KeyValue.parseColumn(cell.getColumn());
167 byte[] column = split[0];
168 byte[] qualifier = split.length > 1 ? split[1] : null;
169 kvs.add(new KeyValue(row.getKey(), column, qualifier,
170 cell.getTimestamp(), cell.getValue()));
171 }
172 results.add(new Result(kvs));
173 }
174 return results.toArray(new Result[results.size()]);
175 }
176
177 protected CellSetModel buildModelFromPut(Put put) {
178 RowModel row = new RowModel(put.getRow());
179 long ts = put.getTimeStamp();
180 for (List<KeyValue> kvs: put.getFamilyMap().values()) {
181 for (KeyValue kv: kvs) {
182 row.addCell(new CellModel(kv.getFamily(), kv.getQualifier(),
183 ts != HConstants.LATEST_TIMESTAMP ? ts : kv.getTimestamp(),
184 kv.getValue()));
185 }
186 }
187 CellSetModel model = new CellSetModel();
188 model.addRow(row);
189 return model;
190 }
191
192
193
194
195
196
197 public RemoteHTable(Client client, String name) {
198 this(client, HBaseConfiguration.create(), Bytes.toBytes(name), null);
199 }
200
201
202
203
204
205
206
207
208 @Deprecated
209 public RemoteHTable(Client client, String name, String accessToken) {
210 this(client, HBaseConfiguration.create(), Bytes.toBytes(name), accessToken);
211 }
212
213
214
215
216
217
218
219 public RemoteHTable(Client client, Configuration conf, String name) {
220 this(client, conf, Bytes.toBytes(name), null);
221 }
222
223
224
225
226
227
228
229
230
231 @Deprecated
232 public RemoteHTable(Client client, Configuration conf, String name,
233 String accessToken) {
234 this(client, conf, Bytes.toBytes(name), accessToken);
235 }
236
237
238
239
240
241
242
243 public RemoteHTable(Client client, Configuration conf, byte[] name) {
244 this(client, conf, name, null);
245 }
246
247
248
249
250
251
252
253
254
255 @Deprecated
256 public RemoteHTable(Client client, Configuration conf, byte[] name,
257 String accessToken) {
258 this.client = client;
259 this.conf = conf;
260 this.name = name;
261 this.maxRetries = conf.getInt("hbase.rest.client.max.retries", 10);
262 this.sleepTime = conf.getLong("hbase.rest.client.sleep", 1000);
263 }
264
265 public byte[] getTableName() {
266 return name.clone();
267 }
268
269 public Configuration getConfiguration() {
270 return conf;
271 }
272
273 public HTableDescriptor getTableDescriptor() throws IOException {
274 StringBuilder sb = new StringBuilder();
275 sb.append('/');
276 sb.append(Bytes.toStringBinary(name));
277 sb.append('/');
278 sb.append("schema");
279 for (int i = 0; i < maxRetries; i++) {
280 Response response = client.get(sb.toString(), Constants.MIMETYPE_PROTOBUF);
281 int code = response.getCode();
282 switch (code) {
283 case 200:
284 TableSchemaModel schema = new TableSchemaModel();
285 schema.getObjectFromMessage(response.getBody());
286 return schema.getTableDescriptor();
287 case 509:
288 try {
289 Thread.sleep(sleepTime);
290 } catch (InterruptedException e) { }
291 break;
292 default:
293 throw new IOException("schema request returned " + code);
294 }
295 }
296 throw new IOException("schema request timed out");
297 }
298
299 public void close() throws IOException {
300 client.shutdown();
301 }
302
303 public Result get(Get get) throws IOException {
304 TimeRange range = get.getTimeRange();
305 String spec = buildRowSpec(get.getRow(), get.getFamilyMap(),
306 range.getMin(), range.getMax(), get.getMaxVersions());
307 if (get.getFilter() != null) {
308 LOG.warn("filters not supported on gets");
309 }
310 Result[] results = getResults(spec);
311 if (results.length > 0) {
312 if (results.length > 1) {
313 LOG.warn("too many results for get (" + results.length + ")");
314 }
315 return results[0];
316 } else {
317 return new Result();
318 }
319 }
320
321 public Result[] get(List<Get> gets) throws IOException {
322 byte[][] rows = new byte[gets.size()][];
323 int maxVersions = 1;
324 int count = 0;
325
326 for (Get g : gets) {
327
328 if (count == 0) {
329 maxVersions = g.getMaxVersions();
330 } else if (g.getMaxVersions() != maxVersions) {
331 LOG.warn("MaxVersions on Gets do not match, using the first in the list ("
332 + maxVersions +")");
333 }
334
335 if (g.getFilter() != null) {
336 LOG.warn("filters not supported on gets");
337 }
338
339 rows[count] = g.getRow();
340 count++;
341 }
342
343 String spec = buildMultiRowSpec(rows, maxVersions);
344
345 return getResults(spec);
346 }
347
348 private Result[] getResults(String spec) throws IOException {
349 for (int i = 0; i < maxRetries; i++) {
350 Response response = client.get(spec, Constants.MIMETYPE_PROTOBUF);
351 int code = response.getCode();
352 switch (code) {
353 case 200:
354 CellSetModel model = new CellSetModel();
355 model.getObjectFromMessage(response.getBody());
356 Result[] results = buildResultFromModel(model);
357 if (results.length > 0) {
358 return results;
359 }
360
361 case 404:
362 return new Result[0];
363
364 case 509:
365 try {
366 Thread.sleep(sleepTime);
367 } catch (InterruptedException e) {
368 }
369 break;
370 default:
371 throw new IOException("get request returned " + code);
372 }
373 }
374 throw new IOException("get request timed out");
375 }
376
377 public boolean exists(Get get) throws IOException {
378 LOG.warn("exists() is really get(), just use get()");
379 Result result = get(get);
380 return (result != null && !(result.isEmpty()));
381 }
382
383 public void put(Put put) throws IOException {
384 CellSetModel model = buildModelFromPut(put);
385 StringBuilder sb = new StringBuilder();
386 sb.append('/');
387 sb.append(Bytes.toStringBinary(name));
388 sb.append('/');
389 sb.append(Bytes.toStringBinary(put.getRow()));
390 for (int i = 0; i < maxRetries; i++) {
391 Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF,
392 model.createProtobufOutput());
393 int code = response.getCode();
394 switch (code) {
395 case 200:
396 return;
397 case 509:
398 try {
399 Thread.sleep(sleepTime);
400 } catch (InterruptedException e) { }
401 break;
402 default:
403 throw new IOException("put request failed with " + code);
404 }
405 }
406 throw new IOException("put request timed out");
407 }
408
409 public void put(List<Put> puts) throws IOException {
410
411
412
413
414 TreeMap<byte[],List<KeyValue>> map =
415 new TreeMap<byte[],List<KeyValue>>(Bytes.BYTES_COMPARATOR);
416 for (Put put: puts) {
417 byte[] row = put.getRow();
418 List<KeyValue> kvs = map.get(row);
419 if (kvs == null) {
420 kvs = new ArrayList<KeyValue>();
421 map.put(row, kvs);
422 }
423 for (List<KeyValue> l: put.getFamilyMap().values()) {
424 kvs.addAll(l);
425 }
426 }
427
428
429 CellSetModel model = new CellSetModel();
430 for (Map.Entry<byte[], List<KeyValue>> e: map.entrySet()) {
431 RowModel row = new RowModel(e.getKey());
432 for (KeyValue kv: e.getValue()) {
433 row.addCell(new CellModel(kv));
434 }
435 model.addRow(row);
436 }
437
438
439 StringBuilder sb = new StringBuilder();
440 sb.append('/');
441 sb.append(Bytes.toStringBinary(name));
442 sb.append("/$multiput");
443 for (int i = 0; i < maxRetries; i++) {
444 Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF,
445 model.createProtobufOutput());
446 int code = response.getCode();
447 switch (code) {
448 case 200:
449 return;
450 case 509:
451 try {
452 Thread.sleep(sleepTime);
453 } catch (InterruptedException e) { }
454 break;
455 default:
456 throw new IOException("multiput request failed with " + code);
457 }
458 }
459 throw new IOException("multiput request timed out");
460 }
461
462 public void delete(Delete delete) throws IOException {
463 String spec = buildRowSpec(delete.getRow(), delete.getFamilyMap(),
464 delete.getTimeStamp(), delete.getTimeStamp(), 1);
465 for (int i = 0; i < maxRetries; i++) {
466 Response response = client.delete(spec);
467 int code = response.getCode();
468 switch (code) {
469 case 200:
470 return;
471 case 509:
472 try {
473 Thread.sleep(sleepTime);
474 } catch (InterruptedException e) { }
475 break;
476 default:
477 throw new IOException("delete request failed with " + code);
478 }
479 }
480 throw new IOException("delete request timed out");
481 }
482
483 public void delete(List<Delete> deletes) throws IOException {
484 for (Delete delete: deletes) {
485 delete(delete);
486 }
487 }
488
489 public void flushCommits() throws IOException {
490
491 }
492
493 class Scanner implements ResultScanner {
494
495 String uri;
496
497 public Scanner(Scan scan) throws IOException {
498 ScannerModel model;
499 try {
500 model = ScannerModel.fromScan(scan);
501 } catch (Exception e) {
502 throw new IOException(e);
503 }
504 StringBuffer sb = new StringBuffer();
505 sb.append('/');
506 sb.append(Bytes.toStringBinary(name));
507 sb.append('/');
508 sb.append("scanner");
509 for (int i = 0; i < maxRetries; i++) {
510 Response response = client.post(sb.toString(),
511 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
512 int code = response.getCode();
513 switch (code) {
514 case 201:
515 uri = response.getLocation();
516 return;
517 case 509:
518 try {
519 Thread.sleep(sleepTime);
520 } catch (InterruptedException e) { }
521 break;
522 default:
523 throw new IOException("scan request failed with " + code);
524 }
525 }
526 throw new IOException("scan request timed out");
527 }
528
529 @Override
530 public Result[] next(int nbRows) throws IOException {
531 StringBuilder sb = new StringBuilder(uri);
532 sb.append("?n=");
533 sb.append(nbRows);
534 for (int i = 0; i < maxRetries; i++) {
535 Response response = client.get(sb.toString(),
536 Constants.MIMETYPE_PROTOBUF);
537 int code = response.getCode();
538 switch (code) {
539 case 200:
540 CellSetModel model = new CellSetModel();
541 model.getObjectFromMessage(response.getBody());
542 return buildResultFromModel(model);
543 case 204:
544 case 206:
545 return null;
546 case 509:
547 try {
548 Thread.sleep(sleepTime);
549 } catch (InterruptedException e) { }
550 break;
551 default:
552 throw new IOException("scanner.next request failed with " + code);
553 }
554 }
555 throw new IOException("scanner.next request timed out");
556 }
557
558 @Override
559 public Result next() throws IOException {
560 Result[] results = next(1);
561 if (results == null || results.length < 1) {
562 return null;
563 }
564 return results[0];
565 }
566
567 class Iter implements Iterator<Result> {
568
569 Result cache;
570
571 public Iter() {
572 try {
573 cache = Scanner.this.next();
574 } catch (IOException e) {
575 LOG.warn(StringUtils.stringifyException(e));
576 }
577 }
578
579 @Override
580 public boolean hasNext() {
581 return cache != null;
582 }
583
584 @Override
585 public Result next() {
586 Result result = cache;
587 try {
588 cache = Scanner.this.next();
589 } catch (IOException e) {
590 LOG.warn(StringUtils.stringifyException(e));
591 cache = null;
592 }
593 return result;
594 }
595
596 @Override
597 public void remove() {
598 throw new RuntimeException("remove() not supported");
599 }
600
601 }
602
603 @Override
604 public Iterator<Result> iterator() {
605 return new Iter();
606 }
607
608 @Override
609 public void close() {
610 try {
611 client.delete(uri);
612 } catch (IOException e) {
613 LOG.warn(StringUtils.stringifyException(e));
614 }
615 }
616
617 }
618
619 public ResultScanner getScanner(Scan scan) throws IOException {
620 return new Scanner(scan);
621 }
622
623 public ResultScanner getScanner(byte[] family) throws IOException {
624 Scan scan = new Scan();
625 scan.addFamily(family);
626 return new Scanner(scan);
627 }
628
629 public ResultScanner getScanner(byte[] family, byte[] qualifier)
630 throws IOException {
631 Scan scan = new Scan();
632 scan.addColumn(family, qualifier);
633 return new Scanner(scan);
634 }
635
636 public boolean isAutoFlush() {
637 return true;
638 }
639
640 public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
641 throw new IOException("getRowOrBefore not supported");
642 }
643
644
645
646
647 public RowLock lockRow(byte[] row) throws IOException {
648 throw new IOException("lockRow not implemented");
649 }
650
651
652
653
654 public void unlockRow(RowLock rl) throws IOException {
655 throw new IOException("unlockRow not implemented");
656 }
657
658 public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
659 byte[] value, Put put) throws IOException {
660
661 put.add(new KeyValue(row, family, qualifier, value));
662
663 CellSetModel model = buildModelFromPut(put);
664 StringBuilder sb = new StringBuilder();
665 sb.append('/');
666 sb.append(Bytes.toStringBinary(name));
667 sb.append('/');
668 sb.append(Bytes.toStringBinary(put.getRow()));
669 sb.append("?check=put");
670
671 for (int i = 0; i < maxRetries; i++) {
672 Response response = client.put(sb.toString(),
673 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
674 int code = response.getCode();
675 switch (code) {
676 case 200:
677 return true;
678 case 304:
679 return false;
680 case 509:
681 try {
682 Thread.sleep(sleepTime);
683 } catch (final InterruptedException e) {
684 }
685 break;
686 default:
687 throw new IOException("checkAndPut request failed with " + code);
688 }
689 }
690 throw new IOException("checkAndPut request timed out");
691 }
692
693 public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
694 byte[] value, Delete delete) throws IOException {
695 Put put = new Put(row);
696
697 put.add(new KeyValue(row, family, qualifier, value));
698 CellSetModel model = buildModelFromPut(put);
699 StringBuilder sb = new StringBuilder();
700 sb.append('/');
701 sb.append(Bytes.toStringBinary(name));
702 sb.append('/');
703 sb.append(Bytes.toStringBinary(row));
704 sb.append("?check=delete");
705
706 for (int i = 0; i < maxRetries; i++) {
707 Response response = client.put(sb.toString(),
708 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
709 int code = response.getCode();
710 switch (code) {
711 case 200:
712 return true;
713 case 304:
714 return false;
715 case 509:
716 try {
717 Thread.sleep(sleepTime);
718 } catch (final InterruptedException e) {
719 }
720 break;
721 default:
722 throw new IOException("checkAndDelete request failed with " + code);
723 }
724 }
725 throw new IOException("checkAndDelete request timed out");
726 }
727
728 public Result increment(Increment increment) throws IOException {
729 throw new IOException("Increment not supported");
730 }
731
732 public Result append(Append append) throws IOException {
733 throw new IOException("Append not supported");
734 }
735
736 public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
737 long amount) throws IOException {
738 throw new IOException("incrementColumnValue not supported");
739 }
740
741 public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier,
742 long amount, boolean writeToWAL) throws IOException {
743 throw new IOException("incrementColumnValue not supported");
744 }
745
746 @Override
747 public void batch(List<? extends Row> actions, Object[] results) throws IOException {
748 throw new IOException("batch not supported");
749 }
750
751 @Override
752 public Object[] batch(List<? extends Row> actions) throws IOException {
753 throw new IOException("batch not supported");
754 }
755
756 @Override
757 public <T extends CoprocessorProtocol> T coprocessorProxy(Class<T> protocol,
758 byte[] row) {
759 throw new
760 UnsupportedOperationException("coprocessorProxy not implemented");
761 }
762
763 @Override
764 public <T extends CoprocessorProtocol, R> Map<byte[], R> coprocessorExec(
765 Class<T> protocol, byte[] startKey, byte[] endKey,
766 Batch.Call<T, R> callable)
767 throws IOException, Throwable {
768 throw new UnsupportedOperationException("coprocessorExec not implemented");
769 }
770
771 @Override
772 public <T extends CoprocessorProtocol, R> void coprocessorExec(
773 Class<T> protocol, byte[] startKey, byte[] endKey,
774 Batch.Call<T, R> callable, Batch.Callback<R> callback)
775 throws IOException, Throwable {
776 throw new UnsupportedOperationException("coprocessorExec not implemented");
777 }
778
779 @Override
780 public void mutateRow(RowMutations rm) throws IOException {
781 throw new IOException("atomicMutation not supported");
782 }
783
784 @Override
785 public void setAutoFlush(boolean autoFlush) {
786 throw new UnsupportedOperationException("setAutoFlush not implemented");
787 }
788
789 @Override
790 public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
791 throw new UnsupportedOperationException("setAutoFlush not implemented");
792 }
793
794 @Override
795 public long getWriteBufferSize() {
796 throw new UnsupportedOperationException("getWriteBufferSize not implemented");
797 }
798
799 @Override
800 public void setWriteBufferSize(long writeBufferSize) throws IOException {
801 throw new IOException("setWriteBufferSize not supported");
802 }
803 }