1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.NavigableMap;
27 import java.util.TreeMap;
28 import java.util.concurrent.atomic.AtomicBoolean;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.fs.PathFilter;
35 import org.apache.hadoop.hbase.Chore;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HServerInfo;
40 import org.apache.hadoop.hbase.RemoteExceptionHandler;
41 import org.apache.hadoop.hbase.UnknownScannerException;
42 import org.apache.hadoop.hbase.client.Delete;
43 import org.apache.hadoop.hbase.client.Get;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.ipc.HRegionInterface;
48 import org.apache.hadoop.hbase.regionserver.HRegion;
49 import org.apache.hadoop.hbase.regionserver.Store;
50 import org.apache.hadoop.hbase.regionserver.StoreFile;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.Writables;
53 import org.apache.hadoop.io.BooleanWritable;
54 import org.apache.hadoop.ipc.RemoteException;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104 abstract class BaseScanner extends Chore {
105 static final Log LOG = LogFactory.getLog(BaseScanner.class.getName());
106
107
108
109 private static final byte[] SPLITA_CHECKED =
110 Bytes.toBytes(Bytes.toString(HConstants.SPLITA_QUALIFIER) + "_checked");
111 private static final byte[] SPLITB_CHECKED =
112 Bytes.toBytes(Bytes.toString(HConstants.SPLITB_QUALIFIER) + "_checked");
113
114 private static byte[] TRUE_WRITABLE_AS_BYTES;
115 static {
116 try {
117 TRUE_WRITABLE_AS_BYTES = Writables.getBytes(new BooleanWritable(true));
118 } catch (IOException e) {
119 e.printStackTrace();
120 }
121 }
122 private final boolean rootRegion;
123 protected final HMaster master;
124
125 protected boolean initialScanComplete;
126
127 protected abstract boolean initialScan();
128 protected abstract void maintenanceScan();
129
130
131
132 final Object scannerLock = new Object();
133
134 BaseScanner(final HMaster master, final boolean rootRegion,
135 final AtomicBoolean stop) {
136 super("Scanner for " + (rootRegion ? "-ROOT-":".META.") + " table",
137 master.getConfiguration().
138 getInt("hbase.master.meta.thread.rescanfrequency", 60 * 1000), stop);
139 this.rootRegion = rootRegion;
140 this.master = master;
141 this.initialScanComplete = false;
142 }
143
144
145 public boolean isInitialScanComplete() {
146 return initialScanComplete;
147 }
148
149 @Override
150 protected boolean initialChore() {
151 return initialScan();
152 }
153
154 @Override
155 protected void chore() {
156 maintenanceScan();
157 }
158
159
160
161
162
163 protected void scanRegion(final MetaRegion region) throws IOException {
164 HRegionInterface regionServer = null;
165 long scannerId = -1L;
166 LOG.info(Thread.currentThread().getName() + " scanning meta region " +
167 region.toString());
168
169
170
171
172 NavigableMap<HRegionInfo, Result> splitParents =
173 new TreeMap<HRegionInfo, Result>();
174 List<byte []> emptyRows = new ArrayList<byte []>();
175 int rows = 0;
176 try {
177 regionServer =
178 this.master.getServerConnection().getHRegionConnection(region.getServer());
179 Scan s = new Scan().addFamily(HConstants.CATALOG_FAMILY);
180
181 s.setCaching(1);
182 scannerId = regionServer.openScanner(region.getRegionName(), s);
183 while (true) {
184 Result values = regionServer.next(scannerId);
185 if (values == null || values.size() == 0) {
186 break;
187 }
188 HRegionInfo info = master.getHRegionInfo(values.getRow(), values);
189 if (info == null) {
190 emptyRows.add(values.getRow());
191 continue;
192 }
193 String serverAddress = getServerAddress(values);
194 long startCode = getStartCode(values);
195
196
197 checkAssigned(regionServer, region, info, serverAddress, startCode, true);
198 if (isSplitParent(info)) {
199 splitParents.put(info, values);
200 }
201 rows += 1;
202 }
203 if (rootRegion) {
204 this.master.getRegionManager().setNumMetaRegions(rows);
205 }
206 } catch (IOException e) {
207 if (e instanceof RemoteException) {
208 e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
209 if (e instanceof UnknownScannerException) {
210
211
212
213 scannerId = -1L;
214 }
215 }
216 throw e;
217 } finally {
218 try {
219 if (scannerId != -1L && regionServer != null) {
220 regionServer.close(scannerId);
221 }
222 } catch (IOException e) {
223 LOG.error("Closing scanner",
224 RemoteExceptionHandler.checkIOException(e));
225 }
226 }
227
228
229
230
231 if (emptyRows.size() > 0) {
232 LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo " +
233 "while scanning meta region " + Bytes.toString(region.getRegionName()));
234 this.master.deleteEmptyMetaRows(regionServer, region.getRegionName(),
235 emptyRows);
236 }
237
238
239
240 if (splitParents.size() > 0) {
241 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
242 HRegionInfo hri = e.getKey();
243 cleanupAndVerifySplits(region.getRegionName(), regionServer,
244 hri, e.getValue());
245 }
246 }
247 LOG.info(Thread.currentThread().getName() + " scan of " + rows +
248 " row(s) of meta region " + region.toString() + " complete");
249 }
250
251
252
253
254
255 static String getServerAddress(final Result r) {
256 final byte[] val = r.getValue(HConstants.CATALOG_FAMILY,
257 HConstants.SERVER_QUALIFIER);
258 return val == null || val.length <= 0 ? "" : Bytes.toString(val);
259 }
260
261
262
263
264
265 static long getStartCode(final Result r) {
266 final byte[] val = r.getValue(HConstants.CATALOG_FAMILY,
267 HConstants.STARTCODE_QUALIFIER);
268 return val == null || val.length <= 0 ? 0L : Bytes.toLong(val);
269 }
270
271
272
273
274
275 private boolean isSplitParent(final HRegionInfo info) {
276 if (!info.isSplit()) {
277 return false;
278 }
279 if (!info.isOffline()) {
280 LOG.warn("Region is split but not offline: " +
281 info.getRegionNameAsString());
282 }
283 return true;
284 }
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300 private boolean cleanupAndVerifySplits(final byte [] metaRegionName,
301 final HRegionInterface srvr, final HRegionInfo parent,
302 Result rowContent)
303 throws IOException {
304 boolean result = false;
305
306 boolean hasReferencesA = checkDaughter(metaRegionName, srvr,
307 parent, rowContent, HConstants.SPLITA_QUALIFIER);
308 boolean hasReferencesB = checkDaughter(metaRegionName, srvr,
309 parent, rowContent, HConstants.SPLITB_QUALIFIER);
310 if (!hasReferencesA && !hasReferencesB) {
311 LOG.info("Deleting region " + parent.getRegionNameAsString() +
312 " because daughter splits no longer hold references");
313 HRegion.deleteRegion(this.master.getFileSystem(),
314 this.master.getRootDir(), parent);
315 HRegion.removeRegionFromMETA(srvr, metaRegionName,
316 parent.getRegionName());
317 result = true;
318 }
319 return result;
320 }
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338 private boolean checkDaughter(final byte [] metaRegionName,
339 final HRegionInterface srvr, final HRegionInfo parent,
340 final Result rowContent, final byte [] qualifier)
341 throws IOException {
342 HRegionInfo hri = getDaughterRegionInfo(rowContent, qualifier);
343 boolean references = hasReferences(metaRegionName, srvr, parent, rowContent,
344 hri, qualifier);
345
346 if (!references) return references;
347 if (!verifyDaughterRowPresent(rowContent, qualifier, srvr, metaRegionName,
348 hri, parent)) {
349
350
351 addDaughterRowChecked(metaRegionName, srvr, parent.getRegionName(), hri,
352 qualifier);
353 }
354 return references;
355 }
356
357
358
359
360
361
362
363
364
365
366
367
368
369 private boolean verifyDaughterRowPresent(final Result rowContent,
370 final byte [] daughter, final HRegionInterface srvr,
371 final byte [] metaRegionName,
372 final HRegionInfo daughterHRI, final HRegionInfo parent)
373 throws IOException {
374
375 boolean present = getDaughterRowChecked(rowContent, daughter);
376 if (present) return present;
377
378
379 byte [] daughterRowKey = daughterHRI.getRegionName();
380 Get g = new Get(daughterRowKey);
381 g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
382 Result r = srvr.get(metaRegionName, g);
383 if (r == null || r.isEmpty()) {
384
385 LOG.warn("Fixup broke split: Add missing split daughter to meta," +
386 " daughter=" + daughterHRI.toString() + ", parent=" + parent.toString());
387 Put p = new Put(daughterRowKey);
388 p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
389 Writables.getBytes(daughterHRI));
390 srvr.put(metaRegionName, p);
391 }
392 return present;
393 }
394
395
396
397
398
399
400
401
402
403
404 private void addDaughterRowChecked(final byte [] metaRegionName,
405 final HRegionInterface srvr, final byte [] parent,
406 final HRegionInfo split, final byte [] daughter)
407 throws IOException {
408 Put p = new Put(parent);
409 p.add(HConstants.CATALOG_FAMILY, getNameOfVerifiedDaughterColumn(daughter),
410 TRUE_WRITABLE_AS_BYTES);
411 srvr.put(metaRegionName, p);
412 }
413
414
415
416
417
418
419
420
421 private boolean getDaughterRowChecked(final Result rowContent,
422 final byte[] which)
423 throws IOException {
424 final byte[] b = rowContent.getValue(HConstants.CATALOG_FAMILY,
425 getNameOfVerifiedDaughterColumn(which));
426 BooleanWritable bw = null;
427 if (b != null && b.length > 0) {
428 bw = (BooleanWritable)Writables.getWritable(b, new BooleanWritable());
429 }
430 return bw == null? false: bw.get();
431 }
432
433
434
435
436
437
438 private static byte [] getNameOfVerifiedDaughterColumn(final byte [] daughter) {
439 return (Bytes.equals(HConstants.SPLITA_QUALIFIER, daughter)
440 ? SPLITA_CHECKED : SPLITB_CHECKED);
441 }
442
443
444
445
446
447
448
449
450
451 private HRegionInfo getDaughterRegionInfo(final Result rowContent,
452 final byte [] which)
453 throws IOException {
454 return Writables.getHRegionInfoOrNull(
455 rowContent.getValue(HConstants.CATALOG_FAMILY, which));
456 }
457
458
459
460
461
462
463
464
465
466
467
468 private void removeDaughterFromParent(final byte [] metaRegionName,
469 final HRegionInterface srvr, final HRegionInfo parent,
470 final HRegionInfo split, final byte [] qualifier)
471 throws IOException {
472 if (LOG.isDebugEnabled()) {
473 LOG.debug(split.getRegionNameAsString() +
474 " no longer has references to " + parent.getRegionNameAsString());
475 }
476 Delete delete = new Delete(parent.getRegionName());
477 delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier);
478 srvr.delete(metaRegionName, delete);
479 }
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494 private boolean hasReferences(final byte [] metaRegionName,
495 final HRegionInterface srvr, final HRegionInfo parent,
496 Result rowContent, final HRegionInfo split, byte [] qualifier)
497 throws IOException {
498 boolean result = false;
499 if (split == null) {
500 return result;
501 }
502 Path tabledir =
503 new Path(this.master.getRootDir(), split.getTableDesc().getNameAsString());
504 for (HColumnDescriptor family: split.getTableDesc().getFamilies()) {
505 Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
506 family.getName());
507 if (!this.master.getFileSystem().exists(p)) continue;
508
509
510 FileStatus [] ps =
511 this.master.getFileSystem().listStatus(p, new PathFilter () {
512 public boolean accept(Path path) {
513 return StoreFile.isReference(path);
514 }
515 }
516 );
517
518 if (ps != null && ps.length > 0) {
519 result = true;
520 break;
521 }
522 }
523 if (!result) {
524 removeDaughterFromParent(metaRegionName, srvr, parent, split, qualifier);
525 }
526 return result;
527 }
528
529
530
531
532
533
534
535
536
537
538
539
540 protected void checkAssigned(final HRegionInterface regionServer,
541 final MetaRegion meta, HRegionInfo info,
542 final String hostnameAndPort, final long startCode, boolean checkTwice)
543 throws IOException {
544 boolean tryAgain = false;
545 String serverName = null;
546 String sa = hostnameAndPort;
547 long sc = startCode;
548 if (sa == null || sa.length() <= 0) {
549
550
551
552 Get g = new Get(info.getRegionName());
553 g.addFamily(HConstants.CATALOG_FAMILY);
554 Result r = regionServer.get(meta.getRegionName(), g);
555 if (r != null && !r.isEmpty()) {
556 sa = getServerAddress(r);
557 sc = getStartCode(r);
558 info = master.getHRegionInfo(r.getRow(), r);
559 }
560 }
561 if (sa != null && sa.length() > 0) {
562 serverName = HServerInfo.getServerName(sa, sc);
563 }
564 HServerInfo storedInfo = null;
565 synchronized (this.master.getRegionManager()) {
566
567
568
569
570 if (info == null || info.isOffline() ||
571 this.master.getRegionManager().regionIsInTransition(info.getRegionNameAsString()) ||
572 (serverName != null && this.master.getServerManager().isDead(serverName))) {
573 return;
574 }
575 if (serverName != null) {
576 storedInfo = this.master.getServerManager().getServerInfo(serverName);
577 }
578
579
580
581 if (storedInfo == null) {
582 if (checkTwice) {
583 tryAgain = true;
584 } else {
585 if (LOG.isDebugEnabled()) {
586 LOG.debug("Current assignment of " + info.getRegionNameAsString() +
587 " is not valid; " + " serverAddress=" + sa +
588 ", startCode=" + sc + " unknown.");
589 }
590
591 this.master.getRegionManager().setUnassigned(info, true);
592 }
593 }
594 }
595 if (tryAgain) {
596
597 if (LOG.isDebugEnabled()) {
598 LOG.debug("Current assignment of " + info.getRegionNameAsString() +
599 " is not valid; " + " serverAddress=" + sa +
600 ", startCode=" + sc + " unknown; checking once more!");
601 }
602
603
604
605 checkAssigned(regionServer, meta, info, null, 0, false);
606 }
607 }
608
609
610
611
612 public void interruptAndStop() {
613 synchronized(scannerLock){
614 if (isAlive()) {
615 super.interrupt();
616 LOG.info("Interrupted");
617 }
618 }
619 }
620 }