1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.Stoppable;
44 import org.apache.hadoop.hbase.catalog.MetaReader;
45 import org.apache.hadoop.hbase.errorhandling.ForeignException;
46 import org.apache.hadoop.hbase.executor.ExecutorService;
47 import org.apache.hadoop.hbase.master.AssignmentManager;
48 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
49 import org.apache.hadoop.hbase.master.MasterFileSystem;
50 import org.apache.hadoop.hbase.master.MasterServices;
51 import org.apache.hadoop.hbase.master.SnapshotSentinel;
52 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
53 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
54 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
55 import org.apache.hadoop.hbase.procedure.Procedure;
56 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
57 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
58 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
59 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
61 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
62 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
63 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
64 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
65 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
66 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
67 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
68 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
69 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
70 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.FSTableDescriptors;
74 import org.apache.hadoop.hbase.util.FSUtils;
75 import org.apache.zookeeper.KeeperException;
76
77
78
79
80
81
82
83
84
85
86 @InterfaceAudience.Private
87 @InterfaceStability.Unstable
88 public class SnapshotManager implements Stoppable {
89 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
90
91
92 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
93
94
95
96
97
98
99
100
101
102
103
104
105 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
106
107
108 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
109
110
111
112
113
114 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
115
116
117 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
118
119
120
121
122
123 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
124
125
126 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
127
128
129 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
130
131
132 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
133
134 private boolean stopped;
135 private final MasterServices master;
136 private final MasterMetrics metricsMaster;
137 private final ProcedureCoordinator coordinator;
138
139
140 private boolean isSnapshotSupported = false;
141
142
143
144
145
146 private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
147
148
149
150
151
152 private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
153
154 private final Path rootDir;
155 private final ExecutorService executorService;
156
157
158
159
160
161 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
162 throws KeeperException, IOException, UnsupportedOperationException {
163 this.master = master;
164 this.metricsMaster = metricsMaster;
165
166 this.rootDir = master.getMasterFileSystem().getRootDir();
167 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
168
169
170 Configuration conf = master.getConfiguration();
171 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
172 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
173 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
174
175
176 String name = master.getServerName().toString();
177 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
178 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
179 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
180 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
181 this.executorService = master.getExecutorService();
182 resetTempDir();
183 }
184
185
186
187
188
189
190
191 public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
192 ProcedureCoordinator coordinator, ExecutorService pool)
193 throws IOException, UnsupportedOperationException {
194 this.master = master;
195 this.metricsMaster = metricsMaster;
196
197 this.rootDir = master.getMasterFileSystem().getRootDir();
198 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
199
200 this.coordinator = coordinator;
201 this.executorService = pool;
202 resetTempDir();
203 }
204
205
206
207
208
209
210 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
211 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
212 }
213
214
215
216
217
218
219
220 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
221 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
222
223 FileSystem fs = master.getMasterFileSystem().getFileSystem();
224 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
225
226
227 if (!fs.exists(snapshotDir)) {
228 return snapshotDescs;
229 }
230
231
232 FileStatus[] snapshots = fs.listStatus(snapshotDir,
233 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
234
235 for (FileStatus snapshot : snapshots) {
236 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
237
238 if (!fs.exists(info)) {
239 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
240 continue;
241 }
242 FSDataInputStream in = null;
243 try {
244 in = fs.open(info);
245 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
246 snapshotDescs.add(desc);
247 } catch (IOException e) {
248 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
249 } finally {
250 if (in != null) {
251 in.close();
252 }
253 }
254 }
255 return snapshotDescs;
256 }
257
258
259
260
261
262
263
264 void resetTempDir() throws IOException {
265
266 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
267 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
268 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
269 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
270 }
271 }
272 }
273
274
275
276
277
278
279
280 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
281
282
283 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
284 if (cpHost != null) {
285 cpHost.preDeleteSnapshot(snapshot);
286 }
287
288
289 if (!isSnapshotCompleted(snapshot)) {
290 throw new SnapshotDoesNotExistException(snapshot);
291 }
292
293 String snapshotName = snapshot.getName();
294 LOG.debug("Deleting snapshot: " + snapshotName);
295
296 MasterFileSystem fs = master.getMasterFileSystem();
297 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
298
299
300 if (!fs.getFileSystem().delete(snapshotDir, true)) {
301 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
302 }
303
304
305 if (cpHost != null) {
306 cpHost.postDeleteSnapshot(snapshot);
307 }
308
309 }
310
311
312
313
314
315
316
317
318
319 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
320
321 if (expected == null) {
322 throw new UnknownSnapshotException(
323 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
324 }
325
326 String ssString = SnapshotDescriptionUtils.toString(expected);
327
328
329
330 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
331
332
333 cleanupSentinels();
334
335 if (handler == null) {
336
337
338
339
340
341
342 if (!isSnapshotCompleted(expected)) {
343 throw new UnknownSnapshotException("Snapshot " + ssString
344 + " is not currently running or one of the known completed snapshots.");
345 }
346
347 return true;
348 }
349
350
351 try {
352 handler.rethrowExceptionIfFailed();
353 } catch (ForeignException e) {
354
355 String status;
356 Procedure p = coordinator.getProcedure(expected.getName());
357 if (p != null) {
358 status = p.getStatus();
359 } else {
360 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
361 }
362 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
363 expected);
364 }
365
366
367 if (handler.isFinished()) {
368 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
369 return true;
370 } else if (LOG.isDebugEnabled()) {
371 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
372 }
373 return false;
374 }
375
376
377
378
379
380
381
382
383
384 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
385 if (isTakingSnapshot(snapshot.getTable())) {
386 return true;
387 }
388 Iterator<Map.Entry<String, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
389 while (it.hasNext()) {
390 Map.Entry<String, SnapshotSentinel> entry = it.next();
391 SnapshotSentinel sentinel = entry.getValue();
392 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
393 return true;
394 }
395 }
396 return false;
397 }
398
399
400
401
402
403
404
405 synchronized boolean isTakingSnapshot(final String tableName) {
406 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
407 return handler != null && !handler.isFinished();
408 }
409
410
411
412
413
414
415
416 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
417 throws HBaseSnapshotException {
418 FileSystem fs = master.getMasterFileSystem().getFileSystem();
419 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
420
421 if (isTakingSnapshot(snapshot)) {
422 SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
423 throw new SnapshotCreationException("Rejected taking "
424 + SnapshotDescriptionUtils.toString(snapshot)
425 + " because we are already running another snapshot "
426 + (handler != null ? ("on the same table " +
427 SnapshotDescriptionUtils.toString(handler.getSnapshot()))
428 : "with the same name"), snapshot);
429 }
430
431
432 if (isRestoringTable(snapshot.getTable())) {
433 SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
434 throw new SnapshotCreationException("Rejected taking "
435 + SnapshotDescriptionUtils.toString(snapshot)
436 + " because we are already have a restore in progress on the same snapshot "
437 + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
438 }
439
440 try {
441
442
443 fs.delete(workingDir, true);
444
445
446 if (!fs.mkdirs(workingDir)) {
447 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
448 + ") for snapshot" , snapshot);
449 }
450 } catch (HBaseSnapshotException e) {
451 throw e;
452 } catch (IOException e) {
453 throw new SnapshotCreationException(
454 "Exception while checking to see if snapshot could be started.", e, snapshot);
455 }
456 }
457
458
459
460
461
462
463 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
464 throws HBaseSnapshotException {
465
466 prepareToTakeSnapshot(snapshot);
467
468
469 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
470
471
472 DisabledTableSnapshotHandler handler =
473 new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
474 snapshotTable(snapshot, handler);
475 }
476
477
478
479
480
481
482 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
483 throws HBaseSnapshotException {
484
485 prepareToTakeSnapshot(snapshot);
486
487
488 EnabledTableSnapshotHandler handler =
489 new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
490 snapshotTable(snapshot, handler);
491 }
492
493
494
495
496
497
498
499
500
501 private synchronized void snapshotTable(SnapshotDescription snapshot,
502 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
503 try {
504 handler.prepare();
505 this.executorService.submit(handler);
506 this.snapshotHandlers.put(snapshot.getTable(), handler);
507 } catch (Exception e) {
508
509 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
510 try {
511 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
512 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
513 SnapshotDescriptionUtils.toString(snapshot));
514 }
515 } catch (IOException e1) {
516 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
517 SnapshotDescriptionUtils.toString(snapshot));
518 }
519
520 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
521 }
522 }
523
524
525
526
527
528
529
530
531 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
532
533 if (isSnapshotCompleted(snapshot)) {
534 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
535 + "' already stored on the filesystem.", snapshot);
536 }
537
538 LOG.debug("No existing snapshot, attempting snapshot...");
539
540
541 cleanupSentinels();
542
543
544 HTableDescriptor desc = null;
545 try {
546 desc = master.getTableDescriptors().get(snapshot.getTable());
547 } catch (FileNotFoundException e) {
548 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
549 LOG.error(msg);
550 throw new SnapshotCreationException(msg, e, snapshot);
551 } catch (IOException e) {
552 throw new SnapshotCreationException("Error while geting table description for table "
553 + snapshot.getTable(), e, snapshot);
554 }
555 if (desc == null) {
556 throw new SnapshotCreationException("Table '" + snapshot.getTable()
557 + "' doesn't exist, can't take snapshot.", snapshot);
558 }
559
560
561 snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
562 .build();
563
564
565 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
566 if (cpHost != null) {
567 cpHost.preSnapshot(snapshot, desc);
568 }
569
570
571 AssignmentManager assignmentMgr = master.getAssignmentManager();
572 if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
573 LOG.debug("Table enabled, starting distributed snapshot.");
574 snapshotEnabledTable(snapshot);
575 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
576 }
577
578 else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
579 LOG.debug("Table is disabled, running snapshot entirely on master.");
580 snapshotDisabledTable(snapshot);
581 LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
582 } else {
583 LOG.error("Can't snapshot table '" + snapshot.getTable()
584 + "', isn't open or closed, we don't know what to do!");
585 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
586 + " isn't fully open.");
587 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
588 }
589
590
591 if (cpHost != null) {
592 cpHost.postSnapshot(snapshot, desc);
593 }
594 }
595
596
597
598
599
600
601
602
603
604
605 public synchronized void setSnapshotHandlerForTesting(final String tableName,
606 final SnapshotSentinel handler) {
607 if (handler != null) {
608 this.snapshotHandlers.put(tableName, handler);
609 } else {
610 this.snapshotHandlers.remove(tableName);
611 }
612 }
613
614
615
616
617 ProcedureCoordinator getCoordinator() {
618 return coordinator;
619 }
620
621
622
623
624
625
626
627
628
629
630
631 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
632 try {
633 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
634 FileSystem fs = master.getMasterFileSystem().getFileSystem();
635
636
637 return fs.exists(snapshotDir);
638 } catch (IllegalArgumentException iae) {
639 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
640 }
641 }
642
643
644
645
646
647
648
649
650 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
651 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
652 String tableName = hTableDescriptor.getNameAsString();
653
654
655 if (isTakingSnapshot(tableName)) {
656 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
657 }
658
659
660 if (isRestoringTable(tableName)) {
661 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
662 }
663
664 try {
665 CloneSnapshotHandler handler =
666 new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
667 this.executorService.submit(handler);
668 this.restoreHandlers.put(tableName, handler);
669 } catch (Exception e) {
670 String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
671 " on table=" + tableName;
672 LOG.error(msg, e);
673 throw new RestoreSnapshotException(msg, e);
674 }
675 }
676
677
678
679
680
681
682 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
683 FileSystem fs = master.getMasterFileSystem().getFileSystem();
684 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
685 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
686
687
688 if (!fs.exists(snapshotDir)) {
689 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
690 throw new SnapshotDoesNotExistException(reqSnapshot);
691 }
692
693
694 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
695 HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
696 String tableName = reqSnapshot.getTable();
697
698
699 cleanupSentinels();
700
701
702 SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, snapshotDir, fsSnapshot);
703
704
705 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
706 if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
707 throw new UnsupportedOperationException("Table '" +
708 fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
709 }
710
711
712 if (cpHost != null) {
713 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
714 }
715 restoreSnapshot(fsSnapshot, snapshotTableDesc);
716 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
717
718 if (cpHost != null) {
719 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
720 }
721 } else {
722 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
723 Bytes.toBytes(tableName));
724 if (cpHost != null) {
725 cpHost.preCloneSnapshot(reqSnapshot, htd);
726 }
727 cloneSnapshot(fsSnapshot, htd);
728 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
729
730 if (cpHost != null) {
731 cpHost.postCloneSnapshot(reqSnapshot, htd);
732 }
733 }
734 }
735
736
737
738
739
740
741
742
743 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
744 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
745 String tableName = hTableDescriptor.getNameAsString();
746
747
748 if (isTakingSnapshot(tableName)) {
749 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
750 }
751
752
753 if (isRestoringTable(tableName)) {
754 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
755 }
756
757 try {
758 RestoreSnapshotHandler handler =
759 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
760 this.executorService.submit(handler);
761 restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
762 } catch (Exception e) {
763 String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
764 snapshot) +
765 " on table=" + tableName;
766 LOG.error(msg, e);
767 throw new RestoreSnapshotException(msg, e);
768 }
769 }
770
771
772
773
774
775
776
777 private synchronized boolean isRestoringTable(final String tableName) {
778 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
779 return(sentinel != null && !sentinel.isFinished());
780 }
781
782
783
784
785
786
787
788
789
790 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
791
792
793 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
794
795
796 cleanupSentinels();
797
798 if (sentinel == null) {
799
800 return true;
801 }
802
803 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
804 + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
805
806
807 sentinel.rethrowExceptionIfFailed();
808
809
810 if (sentinel.isFinished()) {
811 LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
812 " has completed. Notifying the client.");
813 return true;
814 }
815
816 if (LOG.isDebugEnabled()) {
817 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
818 SnapshotDescriptionUtils.toString(snapshot));
819 }
820 return false;
821 }
822
823
824
825
826
827
828
829
830 private synchronized SnapshotSentinel removeSentinelIfFinished(
831 final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
832 SnapshotSentinel h = sentinels.get(snapshot.getTable());
833 if (h == null) {
834 return null;
835 }
836
837 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
838
839 return null;
840 }
841
842
843 if (h.isFinished()) {
844 sentinels.remove(snapshot.getTable());
845 }
846
847 return h;
848 }
849
850
851
852
853
854
855
856
857 private void cleanupSentinels() {
858 cleanupSentinels(this.snapshotHandlers);
859 cleanupSentinels(this.restoreHandlers);
860 }
861
862
863
864
865
866
867 private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
868 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
869 Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
870 while (it.hasNext()) {
871 Map.Entry<String, SnapshotSentinel> entry = it.next();
872 SnapshotSentinel sentinel = entry.getValue();
873 if (sentinel.isFinished() &&
874 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
875 {
876 it.remove();
877 }
878 }
879 }
880
881
882
883
884
885 @Override
886 public void stop(String why) {
887
888 if (this.stopped) return;
889
890 this.stopped = true;
891
892 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
893 snapshotHandler.cancel(why);
894 }
895
896
897 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
898 restoreHandler.cancel(why);
899 }
900
901 try {
902 coordinator.close();
903 } catch (IOException e) {
904 LOG.error("stop ProcedureCoordinator error", e);
905 }
906 }
907
908 @Override
909 public boolean isStopped() {
910 return this.stopped;
911 }
912
913
914
915
916
917
918 public void checkSnapshotSupport() throws UnsupportedOperationException {
919 if (!this.isSnapshotSupported) {
920 throw new UnsupportedOperationException(
921 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
922 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
923 }
924 }
925
926
927
928
929
930
931
932
933
934
935
936 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
937 throws IOException, UnsupportedOperationException {
938
939 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
940 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
941 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
942
943
944 Set<String> hfileCleaners = new HashSet<String>();
945 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
946 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
947
948 Set<String> logCleaners = new HashSet<String>();
949 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
950 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
951
952
953 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
954 FileSystem fs = mfs.getFileSystem();
955 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
956 if (ss != null && !ss.isEmpty()) {
957 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
958 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
959 }
960
961
962
963
964 if (snapshotEnabled) {
965
966 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
967 hfileCleaners.add(HFileLinkCleaner.class.getName());
968 logCleaners.add(SnapshotLogCleaner.class.getName());
969
970
971 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
972 hfileCleaners.toArray(new String[hfileCleaners.size()]));
973 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
974 logCleaners.toArray(new String[logCleaners.size()]));
975 } else {
976
977 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
978 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
979 hfileCleaners.contains(HFileLinkCleaner.class.getName());
980
981
982 if (snapshotEnabled) {
983 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
984 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
985 (userDisabled ? "is set to 'false'." : "is not set."));
986 }
987 }
988
989
990 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
991
992
993
994 if (!snapshotEnabled) {
995 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
996 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
997 if (fs.exists(snapshotDir)) {
998 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
999 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1000 if (snapshots != null) {
1001 LOG.error("Snapshots are present, but cleaners are not enabled.");
1002 checkSnapshotSupport();
1003 }
1004 }
1005 }
1006 }
1007 }