1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.Stoppable;
44 import org.apache.hadoop.hbase.catalog.MetaReader;
45 import org.apache.hadoop.hbase.errorhandling.ForeignException;
46 import org.apache.hadoop.hbase.executor.ExecutorService;
47 import org.apache.hadoop.hbase.master.AssignmentManager;
48 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
49 import org.apache.hadoop.hbase.master.MasterFileSystem;
50 import org.apache.hadoop.hbase.master.MasterServices;
51 import org.apache.hadoop.hbase.master.SnapshotSentinel;
52 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
53 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
54 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
55 import org.apache.hadoop.hbase.procedure.Procedure;
56 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
57 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
58 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
59 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
61 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
62 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
63 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
64 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
65 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
66 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
67 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
68 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
69 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
70 import org.apache.hadoop.hbase.util.Bytes;
71 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
72 import org.apache.hadoop.hbase.util.FSTableDescriptors;
73 import org.apache.hadoop.hbase.util.FSUtils;
74 import org.apache.zookeeper.KeeperException;
75
76
77
78
79
80
81
82
83
84
85 @InterfaceAudience.Private
86 @InterfaceStability.Unstable
87 public class SnapshotManager implements Stoppable {
88 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
89
90
91 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
92
93
94
95
96
97
98
99
100
101
102
103
104 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
105
106
107 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
108
109
110
111
112
113 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
114
115
116 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
117
118
119
120
121
122 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
123
124
125 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
126
127
128 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
129
130
131 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
132
133 private boolean stopped;
134 private final MasterServices master;
135 private final MasterMetrics metricsMaster;
136 private final ProcedureCoordinator coordinator;
137
138
139 private boolean isSnapshotSupported = false;
140
141
142
143
144
145 private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
146
147
148
149
150
151 private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
152
153 private final Path rootDir;
154 private final ExecutorService executorService;
155
156
157
158
159
160 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
161 throws KeeperException, IOException, UnsupportedOperationException {
162 this.master = master;
163 this.metricsMaster = metricsMaster;
164
165 this.rootDir = master.getMasterFileSystem().getRootDir();
166 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
167
168
169 Configuration conf = master.getConfiguration();
170 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
171 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
172 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
173
174
175 String name = master.getServerName().toString();
176 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
177 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
178 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
179 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
180 this.executorService = master.getExecutorService();
181 resetTempDir();
182 }
183
184
185
186
187
188
189
190 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
191 ProcedureCoordinator coordinator, ExecutorService pool)
192 throws IOException, UnsupportedOperationException {
193 this.master = master;
194 this.metricsMaster = metricsMaster;
195
196 this.rootDir = master.getMasterFileSystem().getRootDir();
197 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
198
199 this.coordinator = coordinator;
200 this.executorService = pool;
201 resetTempDir();
202 }
203
204
205
206
207
208
209 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
210 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
211 }
212
213
214
215
216
217
218
219 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
220 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
221
222 FileSystem fs = master.getMasterFileSystem().getFileSystem();
223 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
224
225
226 if (!fs.exists(snapshotDir)) {
227 return snapshotDescs;
228 }
229
230
231 FileStatus[] snapshots = fs.listStatus(snapshotDir,
232 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
233
234 for (FileStatus snapshot : snapshots) {
235 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
236
237 if (!fs.exists(info)) {
238 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
239 continue;
240 }
241 FSDataInputStream in = null;
242 try {
243 in = fs.open(info);
244 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
245 snapshotDescs.add(desc);
246 } catch (IOException e) {
247 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
248 } finally {
249 if (in != null) {
250 in.close();
251 }
252 }
253 }
254 return snapshotDescs;
255 }
256
257
258
259
260
261
262
263 void resetTempDir() throws IOException {
264
265 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
266 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
267 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
268 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
269 }
270 }
271 }
272
273
274
275
276
277
278
279 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
280
281
282 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
283 if (cpHost != null) {
284 cpHost.preDeleteSnapshot(snapshot);
285 }
286
287
288 if (!isSnapshotCompleted(snapshot)) {
289 throw new SnapshotDoesNotExistException(snapshot);
290 }
291
292 String snapshotName = snapshot.getName();
293 LOG.debug("Deleting snapshot: " + snapshotName);
294
295 MasterFileSystem fs = master.getMasterFileSystem();
296 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
297
298
299 if (!fs.getFileSystem().delete(snapshotDir, true)) {
300 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
301 }
302
303
304 if (cpHost != null) {
305 cpHost.postDeleteSnapshot(snapshot);
306 }
307
308 }
309
310
311
312
313
314
315
316
317
318 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
319
320 if (expected == null) {
321 throw new UnknownSnapshotException(
322 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
323 }
324
325 String ssString = SnapshotDescriptionUtils.toString(expected);
326
327
328
329 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
330
331
332 cleanupSentinels();
333
334 if (handler == null) {
335
336
337
338
339
340
341 if (!isSnapshotCompleted(expected)) {
342 throw new UnknownSnapshotException("Snapshot " + ssString
343 + " is not currently running or one of the known completed snapshots.");
344 }
345
346 return true;
347 }
348
349
350 try {
351 handler.rethrowExceptionIfFailed();
352 } catch (ForeignException e) {
353
354 String status;
355 Procedure p = coordinator.getProcedure(expected.getName());
356 if (p != null) {
357 status = p.getStatus();
358 } else {
359 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
360 }
361 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
362 expected);
363 }
364
365
366 if (handler.isFinished()) {
367 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
368 return true;
369 } else if (LOG.isDebugEnabled()) {
370 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
371 }
372 return false;
373 }
374
375
376
377
378
379
380
381
382
383 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
384 if (isTakingSnapshot(snapshot.getTable())) {
385 return true;
386 }
387 Iterator<Map.Entry<String, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
388 while (it.hasNext()) {
389 Map.Entry<String, SnapshotSentinel> entry = it.next();
390 SnapshotSentinel sentinel = entry.getValue();
391 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
392 return true;
393 }
394 }
395 return false;
396 }
397
398
399
400
401
402
403
404 synchronized boolean isTakingSnapshot(final String tableName) {
405 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
406 return handler != null && !handler.isFinished();
407 }
408
409
410
411
412
413
414
415 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
416 throws HBaseSnapshotException {
417 FileSystem fs = master.getMasterFileSystem().getFileSystem();
418 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
419
420 if (isTakingSnapshot(snapshot)) {
421 SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
422 throw new SnapshotCreationException("Rejected taking "
423 + SnapshotDescriptionUtils.toString(snapshot)
424 + " because we are already running another snapshot "
425 + (handler != null ? ("on the same table " +
426 SnapshotDescriptionUtils.toString(handler.getSnapshot()))
427 : "with the same name"), snapshot);
428 }
429
430
431 if (isRestoringTable(snapshot.getTable())) {
432 SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
433 throw new SnapshotCreationException("Rejected taking "
434 + SnapshotDescriptionUtils.toString(snapshot)
435 + " because we are already have a restore in progress on the same snapshot "
436 + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
437 }
438
439 try {
440
441
442 fs.delete(workingDir, true);
443
444
445 if (!fs.mkdirs(workingDir)) {
446 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
447 + ") for snapshot" , snapshot);
448 }
449 } catch (HBaseSnapshotException e) {
450 throw e;
451 } catch (IOException e) {
452 throw new SnapshotCreationException(
453 "Exception while checking to see if snapshot could be started.", e, snapshot);
454 }
455 }
456
457
458
459
460
461
462 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
463 throws HBaseSnapshotException {
464
465 prepareToTakeSnapshot(snapshot);
466
467
468 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
469
470
471 DisabledTableSnapshotHandler handler =
472 new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
473 snapshotTable(snapshot, handler);
474 }
475
476
477
478
479
480
481 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
482 throws HBaseSnapshotException {
483
484 prepareToTakeSnapshot(snapshot);
485
486
487 EnabledTableSnapshotHandler handler =
488 new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
489 snapshotTable(snapshot, handler);
490 }
491
492
493
494
495
496
497
498
499
500 private synchronized void snapshotTable(SnapshotDescription snapshot,
501 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
502 try {
503 handler.prepare();
504 this.executorService.submit(handler);
505 this.snapshotHandlers.put(snapshot.getTable(), handler);
506 } catch (Exception e) {
507
508 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
509 try {
510 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
511 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
512 SnapshotDescriptionUtils.toString(snapshot));
513 }
514 } catch (IOException e1) {
515 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
516 SnapshotDescriptionUtils.toString(snapshot));
517 }
518
519 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
520 }
521 }
522
523
524
525
526
527
528
529
530 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
531
532 if (isSnapshotCompleted(snapshot)) {
533 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
534 + "' already stored on the filesystem.", snapshot);
535 }
536
537 LOG.debug("No existing snapshot, attempting snapshot...");
538
539
540 cleanupSentinels();
541
542
543 HTableDescriptor desc = null;
544 try {
545 desc = master.getTableDescriptors().get(snapshot.getTable());
546 } catch (FileNotFoundException e) {
547 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
548 LOG.error(msg);
549 throw new SnapshotCreationException(msg, e, snapshot);
550 } catch (IOException e) {
551 throw new SnapshotCreationException("Error while geting table description for table "
552 + snapshot.getTable(), e, snapshot);
553 }
554 if (desc == null) {
555 throw new SnapshotCreationException("Table '" + snapshot.getTable()
556 + "' doesn't exist, can't take snapshot.", snapshot);
557 }
558
559
560 snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
561 .build();
562
563
564 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
565 if (cpHost != null) {
566 cpHost.preSnapshot(snapshot, desc);
567 }
568
569
570 AssignmentManager assignmentMgr = master.getAssignmentManager();
571 if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
572 LOG.debug("Table enabled, starting distributed snapshot.");
573 snapshotEnabledTable(snapshot);
574 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
575 }
576
577 else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
578 LOG.debug("Table is disabled, running snapshot entirely on master.");
579 snapshotDisabledTable(snapshot);
580 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
581 } else {
582 LOG.error("Can't snapshot table '" + snapshot.getTable()
583 + "', isn't open or closed, we don't know what to do!");
584 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
585 + " isn't fully open.");
586 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
587 }
588
589
590 if (cpHost != null) {
591 cpHost.postSnapshot(snapshot, desc);
592 }
593 }
594
595
596
597
598
599
600
601
602
603
604 public synchronized void setSnapshotHandlerForTesting(final String tableName,
605 final SnapshotSentinel handler) {
606 if (handler != null) {
607 this.snapshotHandlers.put(tableName, handler);
608 } else {
609 this.snapshotHandlers.remove(tableName);
610 }
611 }
612
613
614
615
616 ProcedureCoordinator getCoordinator() {
617 return coordinator;
618 }
619
620
621
622
623
624
625
626
627
628
629
630 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
631 try {
632 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
633 FileSystem fs = master.getMasterFileSystem().getFileSystem();
634
635
636 return fs.exists(snapshotDir);
637 } catch (IllegalArgumentException iae) {
638 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
639 }
640 }
641
642
643
644
645
646
647
648
649 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
650 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
651 String tableName = hTableDescriptor.getNameAsString();
652
653
654 if (isTakingSnapshot(tableName)) {
655 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
656 }
657
658
659 if (isRestoringTable(tableName)) {
660 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
661 }
662
663 try {
664 CloneSnapshotHandler handler =
665 new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
666 this.executorService.submit(handler);
667 this.restoreHandlers.put(tableName, handler);
668 } catch (Exception e) {
669 String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
670 " on table=" + tableName;
671 LOG.error(msg, e);
672 throw new RestoreSnapshotException(msg, e);
673 }
674 }
675
676
677
678
679
680
681 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
682 FileSystem fs = master.getMasterFileSystem().getFileSystem();
683 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
684 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
685
686
687 if (!fs.exists(snapshotDir)) {
688 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
689 throw new SnapshotDoesNotExistException(reqSnapshot);
690 }
691
692
693 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
694 HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
695 String tableName = reqSnapshot.getTable();
696
697
698 cleanupSentinels();
699
700
701 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
702 if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
703 throw new UnsupportedOperationException("Table '" +
704 fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
705 }
706
707
708 if (cpHost != null) {
709 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
710 }
711 restoreSnapshot(fsSnapshot, snapshotTableDesc);
712 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
713
714 if (cpHost != null) {
715 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
716 }
717 } else {
718 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
719 Bytes.toBytes(tableName));
720 if (cpHost != null) {
721 cpHost.preCloneSnapshot(reqSnapshot, htd);
722 }
723 cloneSnapshot(fsSnapshot, htd);
724 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
725
726 if (cpHost != null) {
727 cpHost.postCloneSnapshot(reqSnapshot, htd);
728 }
729 }
730 }
731
732
733
734
735
736
737
738
739 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
740 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
741 String tableName = hTableDescriptor.getNameAsString();
742
743
744 if (isTakingSnapshot(tableName)) {
745 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
746 }
747
748
749 if (isRestoringTable(tableName)) {
750 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
751 }
752
753 try {
754 RestoreSnapshotHandler handler =
755 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
756 this.executorService.submit(handler);
757 restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
758 } catch (Exception e) {
759 String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
760 snapshot) +
761 " on table=" + tableName;
762 LOG.error(msg, e);
763 throw new RestoreSnapshotException(msg, e);
764 }
765 }
766
767
768
769
770
771
772
773 private synchronized boolean isRestoringTable(final String tableName) {
774 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
775 return(sentinel != null && !sentinel.isFinished());
776 }
777
778
779
780
781
782
783
784
785
786 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
787
788
789 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
790
791
792 cleanupSentinels();
793
794 if (sentinel == null) {
795
796 return true;
797 }
798
799 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
800 + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
801
802
803 sentinel.rethrowExceptionIfFailed();
804
805
806 if (sentinel.isFinished()) {
807 LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
808 " has completed. Notifying the client.");
809 return true;
810 }
811
812 if (LOG.isDebugEnabled()) {
813 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
814 SnapshotDescriptionUtils.toString(snapshot));
815 }
816 return false;
817 }
818
819
820
821
822
823
824
825
826 private synchronized SnapshotSentinel removeSentinelIfFinished(
827 final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
828 SnapshotSentinel h = sentinels.get(snapshot.getTable());
829 if (h == null) {
830 return null;
831 }
832
833 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
834
835 return null;
836 }
837
838
839 if (h.isFinished()) {
840 sentinels.remove(snapshot.getTable());
841 }
842
843 return h;
844 }
845
846
847
848
849
850
851
852
853 private void cleanupSentinels() {
854 cleanupSentinels(this.snapshotHandlers);
855 cleanupSentinels(this.restoreHandlers);
856 }
857
858
859
860
861
862
863 private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
864 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
865 Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
866 while (it.hasNext()) {
867 Map.Entry<String, SnapshotSentinel> entry = it.next();
868 SnapshotSentinel sentinel = entry.getValue();
869 if (sentinel.isFinished() &&
870 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
871 {
872 it.remove();
873 }
874 }
875 }
876
877
878
879
880
881 @Override
882 public void stop(String why) {
883
884 if (this.stopped) return;
885
886 this.stopped = true;
887
888 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
889 snapshotHandler.cancel(why);
890 }
891
892
893 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
894 restoreHandler.cancel(why);
895 }
896
897 try {
898 coordinator.close();
899 } catch (IOException e) {
900 LOG.error("stop ProcedureCoordinator error", e);
901 }
902 }
903
904 @Override
905 public boolean isStopped() {
906 return this.stopped;
907 }
908
909
910
911
912
913
914 public void checkSnapshotSupport() throws UnsupportedOperationException {
915 if (!this.isSnapshotSupported) {
916 throw new UnsupportedOperationException(
917 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
918 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
919 }
920 }
921
922
923
924
925
926
927
928
929
930
931
932 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
933 throws IOException, UnsupportedOperationException {
934
935 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
936 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
937 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
938
939
940 Set<String> hfileCleaners = new HashSet<String>();
941 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
942 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
943
944 Set<String> logCleaners = new HashSet<String>();
945 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
946 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
947
948
949 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
950 FileSystem fs = mfs.getFileSystem();
951 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
952 if (ss != null && !ss.isEmpty()) {
953 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
954 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
955 }
956
957
958
959
960 if (snapshotEnabled) {
961
962 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
963 hfileCleaners.add(HFileLinkCleaner.class.getName());
964 logCleaners.add(SnapshotLogCleaner.class.getName());
965
966
967 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
968 hfileCleaners.toArray(new String[hfileCleaners.size()]));
969 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
970 logCleaners.toArray(new String[logCleaners.size()]));
971 } else {
972
973 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
974 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
975 hfileCleaners.contains(HFileLinkCleaner.class.getName());
976
977
978 if (snapshotEnabled) {
979 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
980 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
981 (userDisabled ? "is set to 'false'." : "is not set."));
982 }
983 }
984
985
986 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
987
988
989
990 if (!snapshotEnabled) {
991 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
992 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
993 if (fs.exists(snapshotDir)) {
994 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
995 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
996 if (snapshots != null) {
997 LOG.error("Snapshots are present, but cleaners are not enabled.");
998 checkSnapshotSupport();
999 }
1000 }
1001 }
1002 }
1003 }