1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import java.io.IOException;
21 import java.io.InterruptedIOException;
22 import java.net.ConnectException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
33 import org.apache.hadoop.hbase.ServerName;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Mutation;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
41 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
42 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45
46 import com.google.protobuf.ServiceException;
47
48
49
50
51
52
53 @InterfaceAudience.Private
54 public class MetaEditor {
55
56
57
58 private static final Log LOG = LogFactory.getLog(MetaEditor.class);
59
60
61
62
63 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
64 throws IOException {
65 return makePutFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
66 }
67
68
69
70
71 public static Put makePutFromRegionInfo(HRegionInfo regionInfo, long ts)
72 throws IOException {
73 Put put = new Put(regionInfo.getRegionName(), ts);
74 addRegionInfo(put, regionInfo);
75 return put;
76 }
77
78
79
80
81
82 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
83 if (regionInfo == null) {
84 throw new IllegalArgumentException("Can't make a delete for null region");
85 }
86 Delete delete = new Delete(regionInfo.getRegionName());
87 return delete;
88 }
89
90
91
92
93 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
94 if (splitA != null) {
95 put.addImmutable(
96 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
97 }
98 if (splitB != null) {
99 put.addImmutable(
100 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
101 }
102 return put;
103 }
104
105
106
107
108
109
110
111 static void putToMetaTable(final CatalogTracker ct, final Put p)
112 throws IOException {
113 put(MetaReader.getMetaHTable(ct), p);
114 }
115
116
117
118
119
120
121
122 static void putToCatalogTable(final CatalogTracker ct, final Put p)
123 throws IOException {
124 put(MetaReader.getCatalogHTable(ct), p);
125 }
126
127
128
129
130
131
132 private static void put(final HTable t, final Put p) throws IOException {
133 try {
134 t.put(p);
135 } finally {
136 t.close();
137 }
138 }
139
140
141
142
143
144
145
146 public static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
147 throws IOException {
148 HTable t = MetaReader.getMetaHTable(ct);
149 try {
150 t.put(ps);
151 } finally {
152 t.close();
153 }
154 }
155
156
157
158
159
160
161
162 static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
163 throws IOException {
164 List<Delete> dels = new ArrayList<Delete>(1);
165 dels.add(d);
166 deleteFromMetaTable(ct, dels);
167 }
168
169
170
171
172
173
174
175 public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
176 throws IOException {
177 HTable t = MetaReader.getMetaHTable(ct);
178 try {
179 t.delete(deletes);
180 } finally {
181 t.close();
182 }
183 }
184
185
186
187
188
189
190
191 public static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
192 throws IOException {
193 HTable t = MetaReader.getMetaHTable(ct);
194 try {
195 t.batch(mutations);
196 } catch (InterruptedException e) {
197 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
198 ie.initCause(e);
199 throw ie;
200 } finally {
201 t.close();
202 }
203 }
204
205
206
207
208
209
210 public static void addRegionToMeta(CatalogTracker catalogTracker,
211 HRegionInfo regionInfo)
212 throws IOException {
213 putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
214 LOG.info("Added " + regionInfo.getRegionNameAsString());
215 }
216
217
218
219
220
221
222
223
224 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo) throws IOException {
225 addRegionToMeta(meta, regionInfo, null, null);
226 }
227
228
229
230
231
232
233
234
235
236
237
238
239
240 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo,
241 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
242 Put put = makePutFromRegionInfo(regionInfo);
243 addDaughtersToPut(put, splitA, splitB);
244 meta.put(put);
245 if (LOG.isDebugEnabled()) {
246 LOG.debug("Added " + regionInfo.getRegionNameAsString());
247 }
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262 public static void addRegionToMeta(CatalogTracker catalogTracker, HRegionInfo regionInfo,
263 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
264 HTable meta = MetaReader.getMetaHTable(catalogTracker);
265 try {
266 addRegionToMeta(meta, regionInfo, splitA, splitB);
267 } finally {
268 meta.close();
269 }
270 }
271
272
273
274
275
276
277
278 public static void addRegionsToMeta(CatalogTracker catalogTracker,
279 List<HRegionInfo> regionInfos)
280 throws IOException {
281 addRegionsToMeta(catalogTracker, regionInfos, HConstants.LATEST_TIMESTAMP);
282 }
283
284
285
286
287
288
289
290
291 public static void addRegionsToMeta(CatalogTracker catalogTracker,
292 List<HRegionInfo> regionInfos, long ts)
293 throws IOException {
294 List<Put> puts = new ArrayList<Put>();
295 for (HRegionInfo regionInfo : regionInfos) {
296 puts.add(makePutFromRegionInfo(regionInfo, ts));
297 }
298 putsToMetaTable(catalogTracker, puts);
299 LOG.info("Added " + puts.size());
300 }
301
302
303
304
305
306
307
308 public static void addDaughter(final CatalogTracker catalogTracker,
309 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
310 throws NotAllMetaRegionsOnlineException, IOException {
311 Put put = new Put(regionInfo.getRegionName());
312 addRegionInfo(put, regionInfo);
313 if (sn != null) {
314 addLocation(put, sn, openSeqNum);
315 }
316 putToMetaTable(catalogTracker, put);
317 LOG.info("Added daughter " + regionInfo.getEncodedName() +
318 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
319 }
320
321
322
323
324
325
326
327
328
329
330
331
332 public static void mergeRegions(final CatalogTracker catalogTracker,
333 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
334 ServerName sn) throws IOException {
335 HTable meta = MetaReader.getMetaHTable(catalogTracker);
336 try {
337 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
338
339
340 Put putOfMerged = makePutFromRegionInfo(copyOfMerged);
341 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
342 regionA.toByteArray());
343 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
344 regionB.toByteArray());
345
346
347 Delete deleteA = makeDeleteFromRegionInfo(regionA);
348 Delete deleteB = makeDeleteFromRegionInfo(regionB);
349
350
351 addLocation(putOfMerged, sn, 1);
352
353 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
354 + HConstants.DELIMITER);
355 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
356 } finally {
357 meta.close();
358 }
359 }
360
361
362
363
364
365
366
367
368
369
370
371
372 public static void splitRegion(final CatalogTracker catalogTracker,
373 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
374 ServerName sn) throws IOException {
375 HTable meta = MetaReader.getMetaHTable(catalogTracker);
376 try {
377 HRegionInfo copyOfParent = new HRegionInfo(parent);
378 copyOfParent.setOffline(true);
379 copyOfParent.setSplit(true);
380
381
382 Put putParent = makePutFromRegionInfo(copyOfParent);
383 addDaughtersToPut(putParent, splitA, splitB);
384
385
386 Put putA = makePutFromRegionInfo(splitA);
387 Put putB = makePutFromRegionInfo(splitB);
388
389 addLocation(putA, sn, 1);
390 addLocation(putB, sn, 1);
391
392 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
393 multiMutate(meta, tableRow, putParent, putA, putB);
394 } finally {
395 meta.close();
396 }
397 }
398
399
400
401
402 private static void multiMutate(HTable table, byte[] row, Mutation... mutations) throws IOException {
403 CoprocessorRpcChannel channel = table.coprocessorService(row);
404 MutateRowsRequest.Builder mmrBuilder = MutateRowsRequest.newBuilder();
405 for (Mutation mutation : mutations) {
406 if (mutation instanceof Put) {
407 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, mutation));
408 } else if (mutation instanceof Delete) {
409 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, mutation));
410 } else {
411 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
412 + mutation.getClass().getName());
413 }
414 }
415
416 MultiRowMutationService.BlockingInterface service =
417 MultiRowMutationService.newBlockingStub(channel);
418 try {
419 service.mutateRows(null, mmrBuilder.build());
420 } catch (ServiceException ex) {
421 ProtobufUtil.toIOException(ex);
422 }
423 }
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442 public static void updateMetaLocation(CatalogTracker catalogTracker,
443 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
444 throws IOException, ConnectException {
445 updateLocation(catalogTracker, regionInfo, sn, openSeqNum);
446 }
447
448
449
450
451
452
453
454
455
456
457
458
459
460 public static void updateRegionLocation(CatalogTracker catalogTracker,
461 HRegionInfo regionInfo, ServerName sn, long updateSeqNum)
462 throws IOException {
463 updateLocation(catalogTracker, regionInfo, sn, updateSeqNum);
464 }
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479 private static void updateLocation(final CatalogTracker catalogTracker,
480 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
481 throws IOException {
482 Put put = new Put(regionInfo.getRegionName());
483 addLocation(put, sn, openSeqNum);
484 putToCatalogTable(catalogTracker, put);
485 LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
486 " with server=" + sn);
487 }
488
489
490
491
492
493
494
495 public static void deleteRegion(CatalogTracker catalogTracker,
496 HRegionInfo regionInfo)
497 throws IOException {
498 Delete delete = new Delete(regionInfo.getRegionName());
499 deleteFromMetaTable(catalogTracker, delete);
500 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
501 }
502
503
504
505
506
507
508
509 public static void deleteRegions(CatalogTracker catalogTracker,
510 List<HRegionInfo> regionsInfo) throws IOException {
511 deleteRegions(catalogTracker, regionsInfo, HConstants.LATEST_TIMESTAMP);
512 }
513
514
515
516
517
518
519
520
521 public static void deleteRegions(CatalogTracker catalogTracker,
522 List<HRegionInfo> regionsInfo, long ts) throws IOException {
523 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
524 for (HRegionInfo hri: regionsInfo) {
525 deletes.add(new Delete(hri.getRegionName(), ts));
526 }
527 deleteFromMetaTable(catalogTracker, deletes);
528 LOG.info("Deleted " + regionsInfo);
529 }
530
531
532
533
534
535
536
537
538 public static void mutateRegions(CatalogTracker catalogTracker,
539 final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
540 throws IOException {
541 List<Mutation> mutation = new ArrayList<Mutation>();
542 if (regionsToRemove != null) {
543 for (HRegionInfo hri: regionsToRemove) {
544 mutation.add(new Delete(hri.getRegionName()));
545 }
546 }
547 if (regionsToAdd != null) {
548 for (HRegionInfo hri: regionsToAdd) {
549 mutation.add(makePutFromRegionInfo(hri));
550 }
551 }
552 mutateMetaTable(catalogTracker, mutation);
553 if (regionsToRemove != null && regionsToRemove.size() > 0) {
554 LOG.debug("Deleted " + regionsToRemove);
555 }
556 if (regionsToAdd != null && regionsToAdd.size() > 0) {
557 LOG.debug("Added " + regionsToAdd);
558 }
559 }
560
561
562
563
564
565
566
567 public static void overwriteRegions(CatalogTracker catalogTracker,
568 List<HRegionInfo> regionInfos) throws IOException {
569
570 long now = EnvironmentEdgeManager.currentTimeMillis();
571 deleteRegions(catalogTracker, regionInfos, now);
572
573
574
575
576
577
578 addRegionsToMeta(catalogTracker, regionInfos, now+1);
579 LOG.info("Overwritten " + regionInfos);
580 }
581
582
583
584
585
586
587
588 public static void deleteMergeQualifiers(CatalogTracker catalogTracker,
589 final HRegionInfo mergedRegion) throws IOException {
590 Delete delete = new Delete(mergedRegion.getRegionName());
591 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
592 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
593 deleteFromMetaTable(catalogTracker, delete);
594 LOG.info("Deleted references in merged region "
595 + mergedRegion.getRegionNameAsString() + ", qualifier="
596 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
597 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
598 }
599
600 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
601 throws IOException {
602 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
603 hri.toByteArray());
604 return p;
605 }
606
607 private static Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
608
609
610 long now = EnvironmentEdgeManager.currentTimeMillis();
611 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, now,
612 Bytes.toBytes(sn.getHostAndPort()));
613 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, now,
614 Bytes.toBytes(sn.getStartcode()));
615 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, now,
616 Bytes.toBytes(openSeqNum));
617 return p;
618 }
619 }