1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.Stoppable;
44 import org.apache.hadoop.hbase.catalog.MetaReader;
45 import org.apache.hadoop.hbase.errorhandling.ForeignException;
46 import org.apache.hadoop.hbase.executor.ExecutorService;
47 import org.apache.hadoop.hbase.master.AssignmentManager;
48 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
49 import org.apache.hadoop.hbase.master.MasterFileSystem;
50 import org.apache.hadoop.hbase.master.MasterServices;
51 import org.apache.hadoop.hbase.master.SnapshotSentinel;
52 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
53 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
54 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
55 import org.apache.hadoop.hbase.procedure.Procedure;
56 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
57 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
58 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
59 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
61 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
62 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
63 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
64 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
65 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
66 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
67 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
68 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
69 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
70 import org.apache.hadoop.hbase.util.Bytes;
71 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
72 import org.apache.hadoop.hbase.util.FSTableDescriptors;
73 import org.apache.hadoop.hbase.util.FSUtils;
74 import org.apache.zookeeper.KeeperException;
75
76
77
78
79
80
81
82
83
84
85 @InterfaceAudience.Private
86 @InterfaceStability.Unstable
87 public class SnapshotManager implements Stoppable {
88 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
89
90
91 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
92
93
94
95
96
97
98
99
100
101
102
103
104 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
105
106
107 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
108
109
110
111
112
113 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
114
115
116 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 5000;
117
118
119
120
121
122 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
123
124
125 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
126
127
128 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
129
130
131 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
132
133 private boolean stopped;
134 private final long wakeFrequency;
135 private final MasterServices master;
136 private final MasterMetrics metricsMaster;
137 private final ProcedureCoordinator coordinator;
138
139
140 private boolean isSnapshotSupported = false;
141
142
143
144
145
146 private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
147
148
149
150
151
152 private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
153
154 private final Path rootDir;
155 private final ExecutorService executorService;
156
157
158
159
160
161 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
162 throws KeeperException, IOException, UnsupportedOperationException {
163 this.master = master;
164 this.metricsMaster = metricsMaster;
165
166 this.rootDir = master.getMasterFileSystem().getRootDir();
167 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
168
169
170 Configuration conf = master.getConfiguration();
171 this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
172 long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
173 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
174
175
176 String name = master.getServerName().toString();
177 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, keepAliveTime, opThreads, wakeFrequency);
178 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
179 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
180 this.coordinator = new ProcedureCoordinator(comms, tpool);
181 this.executorService = master.getExecutorService();
182 resetTempDir();
183 }
184
185
186
187
188
189
190
191 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
192 ProcedureCoordinator coordinator, ExecutorService pool)
193 throws IOException, UnsupportedOperationException {
194 this.master = master;
195 this.metricsMaster = metricsMaster;
196
197 this.rootDir = master.getMasterFileSystem().getRootDir();
198 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
199
200 this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
201 SNAPSHOT_WAKE_MILLIS_DEFAULT);
202 this.coordinator = coordinator;
203 this.executorService = pool;
204 resetTempDir();
205 }
206
207
208
209
210
211
212 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
213 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
214 }
215
216
217
218
219
220
221
222 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
223 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
224
225 FileSystem fs = master.getMasterFileSystem().getFileSystem();
226 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
227
228
229 if (!fs.exists(snapshotDir)) {
230 return snapshotDescs;
231 }
232
233
234 FileStatus[] snapshots = fs.listStatus(snapshotDir,
235 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
236
237 for (FileStatus snapshot : snapshots) {
238 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
239
240 if (!fs.exists(info)) {
241 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
242 continue;
243 }
244 FSDataInputStream in = null;
245 try {
246 in = fs.open(info);
247 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
248 snapshotDescs.add(desc);
249 } catch (IOException e) {
250 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
251 } finally {
252 if (in != null) {
253 in.close();
254 }
255 }
256 }
257 return snapshotDescs;
258 }
259
260
261
262
263
264
265
266 void resetTempDir() throws IOException {
267
268 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
269 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
270 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
271 }
272 }
273
274
275
276
277
278
279
280 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
281
282
283 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
284 if (cpHost != null) {
285 cpHost.preDeleteSnapshot(snapshot);
286 }
287
288
289 if (!isSnapshotCompleted(snapshot)) {
290 throw new SnapshotDoesNotExistException(snapshot);
291 }
292
293 String snapshotName = snapshot.getName();
294 LOG.debug("Deleting snapshot: " + snapshotName);
295
296 MasterFileSystem fs = master.getMasterFileSystem();
297 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
298
299
300 if (!fs.getFileSystem().delete(snapshotDir, true)) {
301 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
302 }
303
304
305 if (cpHost != null) {
306 cpHost.postDeleteSnapshot(snapshot);
307 }
308
309 }
310
311
312
313
314
315
316
317
318
319 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
320
321 if (expected == null) {
322 throw new UnknownSnapshotException(
323 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
324 }
325
326 String ssString = SnapshotDescriptionUtils.toString(expected);
327
328
329
330 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
331
332
333 cleanupSentinels();
334
335 if (handler == null) {
336
337
338
339
340
341
342 if (!isSnapshotCompleted(expected)) {
343 throw new UnknownSnapshotException("Snapshot " + ssString
344 + " is not currently running or one of the known completed snapshots.");
345 }
346
347 return true;
348 }
349
350
351 try {
352 handler.rethrowExceptionIfFailed();
353 } catch (ForeignException e) {
354
355 String status;
356 Procedure p = coordinator.getProcedure(expected.getName());
357 if (p != null) {
358 status = p.getStatus();
359 } else {
360 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
361 }
362 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
363 expected);
364 }
365
366
367 if (handler.isFinished()) {
368 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
369 return true;
370 } else if (LOG.isDebugEnabled()) {
371 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
372 }
373 return false;
374 }
375
376
377
378
379
380
381
382 synchronized boolean isTakingSnapshot(final String tableName) {
383 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
384 return handler != null && !handler.isFinished();
385 }
386
387
388
389
390
391
392
393 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
394 throws HBaseSnapshotException {
395 FileSystem fs = master.getMasterFileSystem().getFileSystem();
396 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
397
398
399 if (isTakingSnapshot(snapshot.getTable())) {
400 SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
401 throw new SnapshotCreationException("Rejected taking "
402 + SnapshotDescriptionUtils.toString(snapshot)
403 + " because we are already running another snapshot "
404 + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
405 }
406
407
408 if (isRestoringTable(snapshot.getTable())) {
409 SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
410 throw new SnapshotCreationException("Rejected taking "
411 + SnapshotDescriptionUtils.toString(snapshot)
412 + " because we are already have a restore in progress on the same snapshot "
413 + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
414 }
415
416 try {
417
418
419 fs.delete(workingDir, true);
420
421
422 if (!fs.mkdirs(workingDir)) {
423 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
424 + ") for snapshot" , snapshot);
425 }
426 } catch (HBaseSnapshotException e) {
427 throw e;
428 } catch (IOException e) {
429 throw new SnapshotCreationException(
430 "Exception while checking to see if snapshot could be started.", e, snapshot);
431 }
432 }
433
434
435
436
437
438
439 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
440 throws HBaseSnapshotException {
441
442 prepareToTakeSnapshot(snapshot);
443
444
445 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
446
447
448 DisabledTableSnapshotHandler handler =
449 new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
450 snapshotTable(snapshot, handler);
451 }
452
453
454
455
456
457
458 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
459 throws HBaseSnapshotException {
460
461 prepareToTakeSnapshot(snapshot);
462
463
464 EnabledTableSnapshotHandler handler =
465 new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
466 snapshotTable(snapshot, handler);
467 }
468
469
470
471
472
473
474
475
476
477 private synchronized void snapshotTable(SnapshotDescription snapshot,
478 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
479 try {
480 handler.prepare();
481 this.executorService.submit(handler);
482 this.snapshotHandlers.put(snapshot.getTable(), handler);
483 } catch (Exception e) {
484
485 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
486 try {
487 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
488 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
489 SnapshotDescriptionUtils.toString(snapshot));
490 }
491 } catch (IOException e1) {
492 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
493 SnapshotDescriptionUtils.toString(snapshot));
494 }
495
496 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
497 }
498 }
499
500
501
502
503
504
505
506
507 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
508
509 if (isSnapshotCompleted(snapshot)) {
510 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
511 + "' already stored on the filesystem.", snapshot);
512 }
513
514 LOG.debug("No existing snapshot, attempting snapshot...");
515
516
517 cleanupSentinels();
518
519
520 HTableDescriptor desc = null;
521 try {
522 desc = master.getTableDescriptors().get(snapshot.getTable());
523 } catch (FileNotFoundException e) {
524 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
525 LOG.error(msg);
526 throw new SnapshotCreationException(msg, e, snapshot);
527 } catch (IOException e) {
528 throw new SnapshotCreationException("Error while geting table description for table "
529 + snapshot.getTable(), e, snapshot);
530 }
531 if (desc == null) {
532 throw new SnapshotCreationException("Table '" + snapshot.getTable()
533 + "' doesn't exist, can't take snapshot.", snapshot);
534 }
535
536
537 snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
538 .build();
539
540
541 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
542 if (cpHost != null) {
543 cpHost.preSnapshot(snapshot, desc);
544 }
545
546
547 AssignmentManager assignmentMgr = master.getAssignmentManager();
548 if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
549 LOG.debug("Table enabled, starting distributed snapshot.");
550 snapshotEnabledTable(snapshot);
551 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
552 }
553
554 else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
555 LOG.debug("Table is disabled, running snapshot entirely on master.");
556 snapshotDisabledTable(snapshot);
557 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
558 } else {
559 LOG.error("Can't snapshot table '" + snapshot.getTable()
560 + "', isn't open or closed, we don't know what to do!");
561 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
562 + " isn't fully open.");
563 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
564 }
565
566
567 if (cpHost != null) {
568 cpHost.postSnapshot(snapshot, desc);
569 }
570 }
571
572
573
574
575
576
577
578
579
580
581 public synchronized void setSnapshotHandlerForTesting(final String tableName,
582 final SnapshotSentinel handler) {
583 if (handler != null) {
584 this.snapshotHandlers.put(tableName, handler);
585 } else {
586 this.snapshotHandlers.remove(tableName);
587 }
588 }
589
590
591
592
593 ProcedureCoordinator getCoordinator() {
594 return coordinator;
595 }
596
597
598
599
600
601
602
603
604
605
606
607 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
608 try {
609 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
610 FileSystem fs = master.getMasterFileSystem().getFileSystem();
611
612
613 return fs.exists(snapshotDir);
614 } catch (IllegalArgumentException iae) {
615 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
616 }
617 }
618
619
620
621
622
623
624
625
626 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
627 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
628 String tableName = hTableDescriptor.getNameAsString();
629
630
631 if (isTakingSnapshot(tableName)) {
632 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
633 }
634
635
636 if (isRestoringTable(tableName)) {
637 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
638 }
639
640 try {
641 CloneSnapshotHandler handler =
642 new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
643 this.executorService.submit(handler);
644 this.restoreHandlers.put(tableName, handler);
645 } catch (Exception e) {
646 String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
647 " on table=" + tableName;
648 LOG.error(msg, e);
649 throw new RestoreSnapshotException(msg, e);
650 }
651 }
652
653
654
655
656
657
658 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
659 FileSystem fs = master.getMasterFileSystem().getFileSystem();
660 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
661 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
662
663
664 if (!fs.exists(snapshotDir)) {
665 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
666 throw new SnapshotDoesNotExistException(reqSnapshot);
667 }
668
669
670 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
671 HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
672 String tableName = reqSnapshot.getTable();
673
674
675 cleanupSentinels();
676
677
678 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
679 if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
680 throw new UnsupportedOperationException("Table '" +
681 fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
682 }
683
684
685 if (cpHost != null) {
686 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
687 }
688 restoreSnapshot(fsSnapshot, snapshotTableDesc);
689 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
690
691 if (cpHost != null) {
692 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
693 }
694 } else {
695 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
696 Bytes.toBytes(tableName));
697 if (cpHost != null) {
698 cpHost.preCloneSnapshot(reqSnapshot, htd);
699 }
700 cloneSnapshot(fsSnapshot, htd);
701 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
702
703 if (cpHost != null) {
704 cpHost.postCloneSnapshot(reqSnapshot, htd);
705 }
706 }
707 }
708
709
710
711
712
713
714
715
716 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
717 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
718 String tableName = hTableDescriptor.getNameAsString();
719
720
721 if (isTakingSnapshot(tableName)) {
722 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
723 }
724
725
726 if (isRestoringTable(tableName)) {
727 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
728 }
729
730 try {
731 RestoreSnapshotHandler handler =
732 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
733 this.executorService.submit(handler);
734 restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
735 } catch (Exception e) {
736 String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
737 snapshot) +
738 " on table=" + tableName;
739 LOG.error(msg, e);
740 throw new RestoreSnapshotException(msg, e);
741 }
742 }
743
744
745
746
747
748
749
750 private synchronized boolean isRestoringTable(final String tableName) {
751 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
752 return(sentinel != null && !sentinel.isFinished());
753 }
754
755
756
757
758
759
760
761
762
763 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
764
765
766 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
767
768
769 cleanupSentinels();
770
771 if (sentinel == null) {
772
773 return true;
774 }
775
776 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
777 + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
778
779
780 sentinel.rethrowExceptionIfFailed();
781
782
783 if (sentinel.isFinished()) {
784 LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
785 " has completed. Notifying the client.");
786 return true;
787 }
788
789 if (LOG.isDebugEnabled()) {
790 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
791 SnapshotDescriptionUtils.toString(snapshot));
792 }
793 return false;
794 }
795
796
797
798
799
800
801
802
803 private synchronized SnapshotSentinel removeSentinelIfFinished(
804 final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
805 SnapshotSentinel h = sentinels.get(snapshot.getTable());
806 if (h == null) {
807 return null;
808 }
809
810 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
811
812 return null;
813 }
814
815
816 if (h.isFinished()) {
817 sentinels.remove(snapshot.getTable());
818 }
819
820 return h;
821 }
822
823
824
825
826
827
828
829
830 private void cleanupSentinels() {
831 cleanupSentinels(this.snapshotHandlers);
832 cleanupSentinels(this.restoreHandlers);
833 }
834
835
836
837
838
839
840 private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
841 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
842 Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
843 while (it.hasNext()) {
844 Map.Entry<String, SnapshotSentinel> entry = it.next();
845 SnapshotSentinel sentinel = entry.getValue();
846 if (sentinel.isFinished() &&
847 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
848 {
849 it.remove();
850 }
851 }
852 }
853
854
855
856
857
858 @Override
859 public void stop(String why) {
860
861 if (this.stopped) return;
862
863 this.stopped = true;
864
865 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
866 snapshotHandler.cancel(why);
867 }
868
869
870 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
871 restoreHandler.cancel(why);
872 }
873 }
874
875 @Override
876 public boolean isStopped() {
877 return this.stopped;
878 }
879
880
881
882
883
884
885 public void checkSnapshotSupport() throws UnsupportedOperationException {
886 if (!this.isSnapshotSupported) {
887 throw new UnsupportedOperationException(
888 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
889 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
890 }
891 }
892
893
894
895
896
897
898
899
900
901
902
903 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
904 throws IOException, UnsupportedOperationException {
905
906 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
907 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
908 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
909
910
911 Set<String> hfileCleaners = new HashSet<String>();
912 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
913 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
914
915 Set<String> logCleaners = new HashSet<String>();
916 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
917 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
918
919
920 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
921 FileSystem fs = mfs.getFileSystem();
922 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
923 if (ss != null && !ss.isEmpty()) {
924 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
925 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
926 }
927
928
929
930
931 if (snapshotEnabled) {
932
933 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
934 hfileCleaners.add(HFileLinkCleaner.class.getName());
935 logCleaners.add(SnapshotLogCleaner.class.getName());
936
937
938 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
939 hfileCleaners.toArray(new String[hfileCleaners.size()]));
940 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
941 logCleaners.toArray(new String[logCleaners.size()]));
942 } else {
943
944 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
945 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
946 hfileCleaners.contains(HFileLinkCleaner.class.getName());
947
948
949 if (snapshotEnabled) {
950 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
951 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
952 (userDisabled ? "is set to 'false'." : "is not set."));
953 }
954 }
955
956
957 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
958
959
960
961 if (!snapshotEnabled) {
962 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
963 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
964 if (fs.exists(snapshotDir)) {
965 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
966 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
967 if (snapshots != null) {
968 LOG.error("Snapshots are present, but cleaners are not enabled.");
969 checkSnapshotSupport();
970 }
971 }
972 }
973 }
974 }