1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import java.io.IOException;
21 import java.io.InterruptedIOException;
22 import java.net.ConnectException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
33 import org.apache.hadoop.hbase.ServerName;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Mutation;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
41 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
42 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45
46 import com.google.protobuf.ServiceException;
47
48
49
50
51
52
53 @InterfaceAudience.Private
54 public class MetaEditor {
55
56
57
58 private static final Log LOG = LogFactory.getLog(MetaEditor.class);
59
60
61
62
63 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
64 throws IOException {
65 return makePutFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
66 }
67
68
69
70
71 public static Put makePutFromRegionInfo(HRegionInfo regionInfo, long ts)
72 throws IOException {
73 Put put = new Put(regionInfo.getRegionName(), ts);
74 addRegionInfo(put, regionInfo);
75 return put;
76 }
77
78
79
80
81
82 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
83 return makeDeleteFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
84 }
85
86
87
88
89
90 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo, long ts) {
91 if (regionInfo == null) {
92 throw new IllegalArgumentException("Can't make a delete for null region");
93 }
94 Delete delete = new Delete(regionInfo.getRegionName(), ts);
95 return delete;
96 }
97
98
99
100
101 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
102 if (splitA != null) {
103 put.addImmutable(
104 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
105 }
106 if (splitB != null) {
107 put.addImmutable(
108 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
109 }
110 return put;
111 }
112
113
114
115
116
117
118
119 static void putToMetaTable(final CatalogTracker ct, final Put p)
120 throws IOException {
121 put(MetaReader.getMetaHTable(ct), p);
122 }
123
124
125
126
127
128
129
130 static void putToCatalogTable(final CatalogTracker ct, final Put p)
131 throws IOException {
132 put(MetaReader.getCatalogHTable(ct), p);
133 }
134
135
136
137
138
139
140 private static void put(final HTable t, final Put p) throws IOException {
141 try {
142 t.put(p);
143 } finally {
144 t.close();
145 }
146 }
147
148
149
150
151
152
153
154 public static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
155 throws IOException {
156 HTable t = MetaReader.getMetaHTable(ct);
157 try {
158 t.put(ps);
159 } finally {
160 t.close();
161 }
162 }
163
164
165
166
167
168
169
170 static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
171 throws IOException {
172 List<Delete> dels = new ArrayList<Delete>(1);
173 dels.add(d);
174 deleteFromMetaTable(ct, dels);
175 }
176
177
178
179
180
181
182
183 public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
184 throws IOException {
185 HTable t = MetaReader.getMetaHTable(ct);
186 try {
187 t.delete(deletes);
188 } finally {
189 t.close();
190 }
191 }
192
193
194
195
196
197
198
199 public static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
200 throws IOException {
201 HTable t = MetaReader.getMetaHTable(ct);
202 try {
203 t.batch(mutations);
204 } catch (InterruptedException e) {
205 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
206 ie.initCause(e);
207 throw ie;
208 } finally {
209 t.close();
210 }
211 }
212
213
214
215
216
217
218 public static void addRegionToMeta(CatalogTracker catalogTracker,
219 HRegionInfo regionInfo)
220 throws IOException {
221 putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
222 LOG.info("Added " + regionInfo.getRegionNameAsString());
223 }
224
225
226
227
228
229
230
231
232 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo) throws IOException {
233 addRegionToMeta(meta, regionInfo, null, null);
234 }
235
236
237
238
239
240
241
242
243
244
245
246
247
248 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo,
249 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
250 Put put = makePutFromRegionInfo(regionInfo);
251 addDaughtersToPut(put, splitA, splitB);
252 meta.put(put);
253 if (LOG.isDebugEnabled()) {
254 LOG.debug("Added " + regionInfo.getRegionNameAsString());
255 }
256 }
257
258
259
260
261
262
263
264
265
266
267
268
269
270 public static void addRegionToMeta(CatalogTracker catalogTracker, HRegionInfo regionInfo,
271 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
272 HTable meta = MetaReader.getMetaHTable(catalogTracker);
273 try {
274 addRegionToMeta(meta, regionInfo, splitA, splitB);
275 } finally {
276 meta.close();
277 }
278 }
279
280
281
282
283
284
285
286 public static void addRegionsToMeta(CatalogTracker catalogTracker,
287 List<HRegionInfo> regionInfos)
288 throws IOException {
289 addRegionsToMeta(catalogTracker, regionInfos, HConstants.LATEST_TIMESTAMP);
290 }
291
292
293
294
295
296
297
298
299 public static void addRegionsToMeta(CatalogTracker catalogTracker,
300 List<HRegionInfo> regionInfos, long ts)
301 throws IOException {
302 List<Put> puts = new ArrayList<Put>();
303 for (HRegionInfo regionInfo : regionInfos) {
304 puts.add(makePutFromRegionInfo(regionInfo, ts));
305 }
306 putsToMetaTable(catalogTracker, puts);
307 LOG.info("Added " + puts.size());
308 }
309
310
311
312
313
314
315
316 public static void addDaughter(final CatalogTracker catalogTracker,
317 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
318 throws NotAllMetaRegionsOnlineException, IOException {
319 Put put = new Put(regionInfo.getRegionName());
320 addRegionInfo(put, regionInfo);
321 if (sn != null) {
322 addLocation(put, sn, openSeqNum, -1);
323 }
324 putToMetaTable(catalogTracker, put);
325 LOG.info("Added daughter " + regionInfo.getEncodedName() +
326 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
327 }
328
329
330
331
332
333
334
335
336
337
338
339
340
341 public static void mergeRegions(final CatalogTracker catalogTracker,
342 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
343 ServerName sn, long masterSystemTime) throws IOException {
344 HTable meta = MetaReader.getMetaHTable(catalogTracker);
345 try {
346 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
347
348
349 long time = Math.max(EnvironmentEdgeManager.currentTimeMillis(), masterSystemTime);
350
351
352 Put putOfMerged = makePutFromRegionInfo(copyOfMerged, time);
353 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
354 regionA.toByteArray());
355 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
356 regionB.toByteArray());
357
358
359 Delete deleteA = makeDeleteFromRegionInfo(regionA, time);
360 Delete deleteB = makeDeleteFromRegionInfo(regionB, time);
361
362
363 addLocation(putOfMerged, sn, 1, time);
364
365 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
366 + HConstants.DELIMITER);
367 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
368 } finally {
369 meta.close();
370 }
371 }
372
373
374
375
376
377
378
379
380
381
382
383
384 public static void splitRegion(final CatalogTracker catalogTracker,
385 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
386 ServerName sn) throws IOException {
387 HTable meta = MetaReader.getMetaHTable(catalogTracker);
388 try {
389 HRegionInfo copyOfParent = new HRegionInfo(parent);
390 copyOfParent.setOffline(true);
391 copyOfParent.setSplit(true);
392
393
394 Put putParent = makePutFromRegionInfo(copyOfParent);
395 addDaughtersToPut(putParent, splitA, splitB);
396
397
398 Put putA = makePutFromRegionInfo(splitA);
399 Put putB = makePutFromRegionInfo(splitB);
400
401 addLocation(putA, sn, 1, -1);
402 addLocation(putB, sn, 1, -1);
403
404 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
405 multiMutate(meta, tableRow, putParent, putA, putB);
406 } finally {
407 meta.close();
408 }
409 }
410
411
412
413
414 private static void multiMutate(HTable table, byte[] row, Mutation... mutations) throws IOException {
415 CoprocessorRpcChannel channel = table.coprocessorService(row);
416 MutateRowsRequest.Builder mmrBuilder = MutateRowsRequest.newBuilder();
417 for (Mutation mutation : mutations) {
418 if (mutation instanceof Put) {
419 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, mutation));
420 } else if (mutation instanceof Delete) {
421 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, mutation));
422 } else {
423 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
424 + mutation.getClass().getName());
425 }
426 }
427
428 MultiRowMutationService.BlockingInterface service =
429 MultiRowMutationService.newBlockingStub(channel);
430 try {
431 service.mutateRows(null, mmrBuilder.build());
432 } catch (ServiceException ex) {
433 ProtobufUtil.toIOException(ex);
434 }
435 }
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454 public static void updateMetaLocation(CatalogTracker catalogTracker,
455 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
456 throws IOException, ConnectException {
457 updateLocation(catalogTracker, regionInfo, sn, openSeqNum, -1);
458 }
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473 public static void updateRegionLocation(CatalogTracker catalogTracker,
474 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
475 throws IOException {
476 updateLocation(catalogTracker, regionInfo, sn, openSeqNum, -1);
477 }
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493 public static void updateRegionLocation(CatalogTracker catalogTracker,
494 HRegionInfo regionInfo, ServerName sn, long openSeqNum, long masterSystemTime)
495 throws IOException {
496 updateLocation(catalogTracker, regionInfo, sn, openSeqNum, masterSystemTime);
497 }
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513 private static void updateLocation(final CatalogTracker catalogTracker,
514 HRegionInfo regionInfo, ServerName sn, long openSeqNum, long masterSystemTime)
515 throws IOException {
516
517
518 long time = Math.max(EnvironmentEdgeManager.currentTimeMillis(), masterSystemTime);
519
520 Put put = new Put(regionInfo.getRegionName(), time);
521 addLocation(put, sn, openSeqNum, time);
522 putToCatalogTable(catalogTracker, put);
523 LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
524 " with server=" + sn);
525 }
526
527
528
529
530
531
532
533 public static void deleteRegion(CatalogTracker catalogTracker,
534 HRegionInfo regionInfo)
535 throws IOException {
536 Delete delete = new Delete(regionInfo.getRegionName());
537 deleteFromMetaTable(catalogTracker, delete);
538 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
539 }
540
541
542
543
544
545
546
547 public static void deleteRegions(CatalogTracker catalogTracker,
548 List<HRegionInfo> regionsInfo) throws IOException {
549 deleteRegions(catalogTracker, regionsInfo, HConstants.LATEST_TIMESTAMP);
550 }
551
552
553
554
555
556
557
558
559 public static void deleteRegions(CatalogTracker catalogTracker,
560 List<HRegionInfo> regionsInfo, long ts) throws IOException {
561 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
562 for (HRegionInfo hri: regionsInfo) {
563 deletes.add(new Delete(hri.getRegionName(), ts));
564 }
565 deleteFromMetaTable(catalogTracker, deletes);
566 LOG.info("Deleted " + regionsInfo);
567 }
568
569
570
571
572
573
574
575
576 public static void mutateRegions(CatalogTracker catalogTracker,
577 final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
578 throws IOException {
579 List<Mutation> mutation = new ArrayList<Mutation>();
580 if (regionsToRemove != null) {
581 for (HRegionInfo hri: regionsToRemove) {
582 mutation.add(new Delete(hri.getRegionName()));
583 }
584 }
585 if (regionsToAdd != null) {
586 for (HRegionInfo hri: regionsToAdd) {
587 mutation.add(makePutFromRegionInfo(hri));
588 }
589 }
590 mutateMetaTable(catalogTracker, mutation);
591 if (regionsToRemove != null && regionsToRemove.size() > 0) {
592 LOG.debug("Deleted " + regionsToRemove);
593 }
594 if (regionsToAdd != null && regionsToAdd.size() > 0) {
595 LOG.debug("Added " + regionsToAdd);
596 }
597 }
598
599
600
601
602
603
604
605 public static void overwriteRegions(CatalogTracker catalogTracker,
606 List<HRegionInfo> regionInfos) throws IOException {
607
608 long now = EnvironmentEdgeManager.currentTimeMillis();
609 deleteRegions(catalogTracker, regionInfos, now);
610
611
612
613
614
615
616 addRegionsToMeta(catalogTracker, regionInfos, now+1);
617 LOG.info("Overwritten " + regionInfos);
618 }
619
620
621
622
623
624
625
626 public static void deleteMergeQualifiers(CatalogTracker catalogTracker,
627 final HRegionInfo mergedRegion) throws IOException {
628 Delete delete = new Delete(mergedRegion.getRegionName());
629 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
630 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
631 deleteFromMetaTable(catalogTracker, delete);
632 LOG.info("Deleted references in merged region "
633 + mergedRegion.getRegionNameAsString() + ", qualifier="
634 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
635 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
636 }
637
638 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
639 throws IOException {
640 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
641 hri.toByteArray());
642 return p;
643 }
644
645 private static Put addLocation(final Put p, final ServerName sn, long openSeqNum, long time) {
646 if (time <= 0) {
647 time = EnvironmentEdgeManager.currentTimeMillis();
648 }
649 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, time,
650 Bytes.toBytes(sn.getHostAndPort()));
651 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, time,
652 Bytes.toBytes(sn.getStartcode()));
653 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, time,
654 Bytes.toBytes(openSeqNum));
655 return p;
656 }
657 }