1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.Stoppable;
46 import org.apache.hadoop.hbase.MetaTableAccessor;
47 import org.apache.hadoop.hbase.errorhandling.ForeignException;
48 import org.apache.hadoop.hbase.executor.ExecutorService;
49 import org.apache.hadoop.hbase.master.AssignmentManager;
50 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
51 import org.apache.hadoop.hbase.master.MasterFileSystem;
52 import org.apache.hadoop.hbase.master.MasterServices;
53 import org.apache.hadoop.hbase.master.MetricsMaster;
54 import org.apache.hadoop.hbase.master.SnapshotSentinel;
55 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
56 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
57 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
58 import org.apache.hadoop.hbase.procedure.Procedure;
59 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
60 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
61 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
62 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
63 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
64 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
65 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
66 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
67 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
68 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
69 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
70 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
71 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
72 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
73 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
74 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
75 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
76 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
77 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
78 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
79 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
80 import org.apache.hadoop.hbase.util.FSUtils;
81 import org.apache.zookeeper.KeeperException;
82
83
84
85
86
87
88
89
90
91
92 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
93 @InterfaceStability.Unstable
94 public class SnapshotManager extends MasterProcedureManager implements Stoppable {
95 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
96
97
98 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
99
100
101
102
103
104
105
106
107
108
109
110
111 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
112
113
114 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
115
116
117
118
119
120 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
121
122
123 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
124
125
126
127
128
129 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
130
131
132 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
133
134
135 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
136
137
138 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
139
140 private boolean stopped;
141 private MasterServices master;
142 private ProcedureCoordinator coordinator;
143
144
145 private boolean isSnapshotSupported = false;
146
147
148
149
150
151 private Map<TableName, SnapshotSentinel> snapshotHandlers =
152 new HashMap<TableName, SnapshotSentinel>();
153
154
155
156
157
158 private Map<TableName, SnapshotSentinel> restoreHandlers =
159 new HashMap<TableName, SnapshotSentinel>();
160
161 private Path rootDir;
162 private ExecutorService executorService;
163
164 public SnapshotManager() {}
165
166
167
168
169
170
171
172 public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
173 ProcedureCoordinator coordinator, ExecutorService pool)
174 throws IOException, UnsupportedOperationException {
175 this.master = master;
176
177 this.rootDir = master.getMasterFileSystem().getRootDir();
178 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
179
180 this.coordinator = coordinator;
181 this.executorService = pool;
182 resetTempDir();
183 }
184
185
186
187
188
189
190 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
191 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
192 }
193
194
195
196
197
198
199
200 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
201 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
202
203 FileSystem fs = master.getMasterFileSystem().getFileSystem();
204 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
205
206
207 if (!fs.exists(snapshotDir)) {
208 return snapshotDescs;
209 }
210
211
212 FileStatus[] snapshots = fs.listStatus(snapshotDir,
213 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
214
215 for (FileStatus snapshot : snapshots) {
216 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
217
218 if (!fs.exists(info)) {
219 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
220 continue;
221 }
222 FSDataInputStream in = null;
223 try {
224 in = fs.open(info);
225 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
226 snapshotDescs.add(desc);
227 } catch (IOException e) {
228 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
229 } finally {
230 if (in != null) {
231 in.close();
232 }
233 }
234 }
235 return snapshotDescs;
236 }
237
238
239
240
241
242
243
244 void resetTempDir() throws IOException {
245
246 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
247 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
248 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
249 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
250 }
251 }
252 }
253
254
255
256
257
258
259
260 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
261
262
263 MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
264 if (cpHost != null) {
265 cpHost.preDeleteSnapshot(snapshot);
266 }
267
268
269 if (!isSnapshotCompleted(snapshot)) {
270 throw new SnapshotDoesNotExistException(snapshot);
271 }
272
273 String snapshotName = snapshot.getName();
274 LOG.debug("Deleting snapshot: " + snapshotName);
275
276 MasterFileSystem fs = master.getMasterFileSystem();
277 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
278
279
280 if (!fs.getFileSystem().delete(snapshotDir, true)) {
281 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
282 }
283
284
285 if (cpHost != null) {
286 cpHost.postDeleteSnapshot(snapshot);
287 }
288
289 }
290
291
292
293
294
295
296
297
298
299 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
300
301 if (expected == null) {
302 throw new UnknownSnapshotException(
303 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
304 }
305
306 String ssString = ClientSnapshotDescriptionUtils.toString(expected);
307
308
309
310 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
311
312
313 cleanupSentinels();
314
315 if (handler == null) {
316
317
318
319
320
321
322 if (!isSnapshotCompleted(expected)) {
323 throw new UnknownSnapshotException("Snapshot " + ssString
324 + " is not currently running or one of the known completed snapshots.");
325 }
326
327 return true;
328 }
329
330
331 try {
332 handler.rethrowExceptionIfFailed();
333 } catch (ForeignException e) {
334
335 String status;
336 Procedure p = coordinator.getProcedure(expected.getName());
337 if (p != null) {
338 status = p.getStatus();
339 } else {
340 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
341 }
342 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
343 expected);
344 }
345
346
347 if (handler.isFinished()) {
348 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
349 return true;
350 } else if (LOG.isDebugEnabled()) {
351 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
352 }
353 return false;
354 }
355
356
357
358
359
360
361
362
363
364 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
365 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
366 if (isTakingSnapshot(snapshotTable)) {
367 return true;
368 }
369 Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
370 while (it.hasNext()) {
371 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
372 SnapshotSentinel sentinel = entry.getValue();
373 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
374 return true;
375 }
376 }
377 return false;
378 }
379
380
381
382
383
384
385
386 synchronized boolean isTakingSnapshot(final TableName tableName) {
387 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
388 return handler != null && !handler.isFinished();
389 }
390
391
392
393
394
395
396
397 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
398 throws HBaseSnapshotException {
399 FileSystem fs = master.getMasterFileSystem().getFileSystem();
400 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
401 TableName snapshotTable =
402 TableName.valueOf(snapshot.getTable());
403
404
405 if (isTakingSnapshot(snapshot)) {
406 SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
407 throw new SnapshotCreationException("Rejected taking "
408 + ClientSnapshotDescriptionUtils.toString(snapshot)
409 + " because we are already running another snapshot "
410 + (handler != null ? ("on the same table " +
411 ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
412 : "with the same name"), snapshot);
413 }
414
415
416 if (isRestoringTable(snapshotTable)) {
417 SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
418 throw new SnapshotCreationException("Rejected taking "
419 + ClientSnapshotDescriptionUtils.toString(snapshot)
420 + " because we are already have a restore in progress on the same snapshot "
421 + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
422 }
423
424 try {
425
426
427 fs.delete(workingDir, true);
428
429
430 if (!fs.mkdirs(workingDir)) {
431 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
432 + ") for snapshot" , snapshot);
433 }
434 } catch (HBaseSnapshotException e) {
435 throw e;
436 } catch (IOException e) {
437 throw new SnapshotCreationException(
438 "Exception while checking to see if snapshot could be started.", e, snapshot);
439 }
440 }
441
442
443
444
445
446
447 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
448 throws HBaseSnapshotException {
449
450 prepareToTakeSnapshot(snapshot);
451
452
453 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
454
455
456 DisabledTableSnapshotHandler handler =
457 new DisabledTableSnapshotHandler(snapshot, master);
458 snapshotTable(snapshot, handler);
459 }
460
461
462
463
464
465
466 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
467 throws HBaseSnapshotException {
468
469 prepareToTakeSnapshot(snapshot);
470
471
472 EnabledTableSnapshotHandler handler =
473 new EnabledTableSnapshotHandler(snapshot, master, this);
474 snapshotTable(snapshot, handler);
475 }
476
477
478
479
480
481
482
483
484
485 private synchronized void snapshotTable(SnapshotDescription snapshot,
486 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
487 try {
488 handler.prepare();
489 this.executorService.submit(handler);
490 this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
491 } catch (Exception e) {
492
493 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
494 try {
495 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
496 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
497 ClientSnapshotDescriptionUtils.toString(snapshot));
498 }
499 } catch (IOException e1) {
500 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
501 ClientSnapshotDescriptionUtils.toString(snapshot));
502 }
503
504 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
505 }
506 }
507
508
509
510
511
512
513
514
515 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
516
517 if (isSnapshotCompleted(snapshot)) {
518 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
519 + "' already stored on the filesystem.", snapshot);
520 }
521
522 LOG.debug("No existing snapshot, attempting snapshot...");
523
524
525 cleanupSentinels();
526
527
528 HTableDescriptor desc = null;
529 try {
530 desc = master.getTableDescriptors().get(
531 TableName.valueOf(snapshot.getTable()));
532 } catch (FileNotFoundException e) {
533 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
534 LOG.error(msg);
535 throw new SnapshotCreationException(msg, e, snapshot);
536 } catch (IOException e) {
537 throw new SnapshotCreationException("Error while geting table description for table "
538 + snapshot.getTable(), e, snapshot);
539 }
540 if (desc == null) {
541 throw new SnapshotCreationException("Table '" + snapshot.getTable()
542 + "' doesn't exist, can't take snapshot.", snapshot);
543 }
544
545
546 if (!snapshot.hasVersion()) {
547 snapshot = snapshot.toBuilder()
548 .setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
549 .build();
550 }
551
552
553 MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
554 if (cpHost != null) {
555 cpHost.preSnapshot(snapshot, desc);
556 }
557
558
559 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
560 AssignmentManager assignmentMgr = master.getAssignmentManager();
561 if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
562 ZooKeeperProtos.Table.State.ENABLED)) {
563 LOG.debug("Table enabled, starting distributed snapshot.");
564 snapshotEnabledTable(snapshot);
565 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
566 }
567
568 else if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
569 ZooKeeperProtos.Table.State.DISABLED)) {
570 LOG.debug("Table is disabled, running snapshot entirely on master.");
571 snapshotDisabledTable(snapshot);
572 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
573 } else {
574 LOG.error("Can't snapshot table '" + snapshot.getTable()
575 + "', isn't open or closed, we don't know what to do!");
576 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
577 + " isn't fully open.");
578 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
579 }
580
581
582 if (cpHost != null) {
583 cpHost.postSnapshot(snapshot, desc);
584 }
585 }
586
587
588
589
590
591
592
593
594
595
596 public synchronized void setSnapshotHandlerForTesting(
597 final TableName tableName,
598 final SnapshotSentinel handler) {
599 if (handler != null) {
600 this.snapshotHandlers.put(tableName, handler);
601 } else {
602 this.snapshotHandlers.remove(tableName);
603 }
604 }
605
606
607
608
609 ProcedureCoordinator getCoordinator() {
610 return coordinator;
611 }
612
613
614
615
616
617
618
619
620
621
622
623 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
624 try {
625 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
626 FileSystem fs = master.getMasterFileSystem().getFileSystem();
627
628 return fs.exists(snapshotDir);
629 } catch (IllegalArgumentException iae) {
630 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
631 }
632 }
633
634
635
636
637
638
639
640
641 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
642 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
643 TableName tableName = hTableDescriptor.getTableName();
644
645
646 if (isTakingSnapshot(tableName)) {
647 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
648 }
649
650
651 if (isRestoringTable(tableName)) {
652 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
653 }
654
655 try {
656 CloneSnapshotHandler handler =
657 new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
658 this.executorService.submit(handler);
659 this.restoreHandlers.put(tableName, handler);
660 } catch (Exception e) {
661 String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
662 " on table=" + tableName;
663 LOG.error(msg, e);
664 throw new RestoreSnapshotException(msg, e);
665 }
666 }
667
668
669
670
671
672
673 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
674 FileSystem fs = master.getMasterFileSystem().getFileSystem();
675 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
676 MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
677
678
679 if (!fs.exists(snapshotDir)) {
680 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
681 throw new SnapshotDoesNotExistException(reqSnapshot);
682 }
683
684
685 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
686 SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
687 snapshotDir, fsSnapshot);
688 HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
689 TableName tableName = TableName.valueOf(reqSnapshot.getTable());
690
691
692 cleanupSentinels();
693
694
695 SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
696
697
698 if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
699 if (master.getAssignmentManager().getTableStateManager().isTableState(
700 TableName.valueOf(fsSnapshot.getTable()), ZooKeeperProtos.Table.State.ENABLED)) {
701 throw new UnsupportedOperationException("Table '" +
702 TableName.valueOf(fsSnapshot.getTable()) + "' must be disabled in order to " +
703 "perform a restore operation" +
704 ".");
705 }
706
707
708 if (cpHost != null) {
709 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
710 }
711 restoreSnapshot(fsSnapshot, snapshotTableDesc);
712 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
713
714 if (cpHost != null) {
715 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
716 }
717 } else {
718 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
719 if (cpHost != null) {
720 cpHost.preCloneSnapshot(reqSnapshot, htd);
721 }
722 cloneSnapshot(fsSnapshot, htd);
723 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
724
725 if (cpHost != null) {
726 cpHost.postCloneSnapshot(reqSnapshot, htd);
727 }
728 }
729 }
730
731
732
733
734
735
736
737
738 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
739 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
740 TableName tableName = hTableDescriptor.getTableName();
741
742
743 if (isTakingSnapshot(tableName)) {
744 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
745 }
746
747
748 if (isRestoringTable(tableName)) {
749 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
750 }
751
752 try {
753 RestoreSnapshotHandler handler =
754 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
755 this.executorService.submit(handler);
756 restoreHandlers.put(tableName, handler);
757 } catch (Exception e) {
758 String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
759 snapshot) +
760 " on table=" + tableName;
761 LOG.error(msg, e);
762 throw new RestoreSnapshotException(msg, e);
763 }
764 }
765
766
767
768
769
770
771
772 private synchronized boolean isRestoringTable(final TableName tableName) {
773 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
774 return(sentinel != null && !sentinel.isFinished());
775 }
776
777
778
779
780
781
782
783
784
785 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
786
787
788 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
789
790
791 cleanupSentinels();
792
793 if (sentinel == null) {
794
795 return true;
796 }
797
798 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
799 + sentinel.getSnapshot().getName() + " table=" +
800 TableName.valueOf(snapshot.getTable()));
801
802
803 sentinel.rethrowExceptionIfFailed();
804
805
806 if (sentinel.isFinished()) {
807 LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
808 " has completed. Notifying the client.");
809 return true;
810 }
811
812 if (LOG.isDebugEnabled()) {
813 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
814 ClientSnapshotDescriptionUtils.toString(snapshot));
815 }
816 return false;
817 }
818
819
820
821
822
823
824
825
826 private synchronized SnapshotSentinel removeSentinelIfFinished(
827 final Map<TableName, SnapshotSentinel> sentinels,
828 final SnapshotDescription snapshot) {
829 if (!snapshot.hasTable()) {
830 return null;
831 }
832
833 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
834 SnapshotSentinel h = sentinels.get(snapshotTable);
835 if (h == null) {
836 return null;
837 }
838
839 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
840
841 return null;
842 }
843
844
845 if (h.isFinished()) {
846 sentinels.remove(snapshotTable);
847 }
848
849 return h;
850 }
851
852
853
854
855
856
857
858
859 private void cleanupSentinels() {
860 cleanupSentinels(this.snapshotHandlers);
861 cleanupSentinels(this.restoreHandlers);
862 }
863
864
865
866
867
868
869 private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
870 long currentTime = EnvironmentEdgeManager.currentTime();
871 Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
872 sentinels.entrySet().iterator();
873 while (it.hasNext()) {
874 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
875 SnapshotSentinel sentinel = entry.getValue();
876 if (sentinel.isFinished() &&
877 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
878 {
879 it.remove();
880 }
881 }
882 }
883
884
885
886
887
888 @Override
889 public void stop(String why) {
890
891 if (this.stopped) return;
892
893 this.stopped = true;
894
895 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
896 snapshotHandler.cancel(why);
897 }
898
899
900 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
901 restoreHandler.cancel(why);
902 }
903 try {
904 if (coordinator != null) {
905 coordinator.close();
906 }
907 } catch (IOException e) {
908 LOG.error("stop ProcedureCoordinator error", e);
909 }
910 }
911
912 @Override
913 public boolean isStopped() {
914 return this.stopped;
915 }
916
917
918
919
920
921
922 public void checkSnapshotSupport() throws UnsupportedOperationException {
923 if (!this.isSnapshotSupported) {
924 throw new UnsupportedOperationException(
925 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
926 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
927 }
928 }
929
930
931
932
933
934
935
936
937
938
939
940 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
941 throws IOException, UnsupportedOperationException {
942
943 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
944 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
945 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
946
947
948 Set<String> hfileCleaners = new HashSet<String>();
949 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
950 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
951
952 Set<String> logCleaners = new HashSet<String>();
953 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
954 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
955
956
957 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
958 FileSystem fs = mfs.getFileSystem();
959 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
960 if (ss != null && !ss.isEmpty()) {
961 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
962 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
963 }
964
965
966
967
968 if (snapshotEnabled) {
969
970 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
971 hfileCleaners.add(HFileLinkCleaner.class.getName());
972 logCleaners.add(SnapshotLogCleaner.class.getName());
973
974
975 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
976 hfileCleaners.toArray(new String[hfileCleaners.size()]));
977 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
978 logCleaners.toArray(new String[logCleaners.size()]));
979 } else {
980
981 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
982 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
983 hfileCleaners.contains(HFileLinkCleaner.class.getName());
984
985
986 if (snapshotEnabled) {
987 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
988 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
989 (userDisabled ? "is set to 'false'." : "is not set."));
990 }
991 }
992
993
994 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
995
996
997
998 if (!snapshotEnabled) {
999 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
1000 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
1001 if (fs.exists(snapshotDir)) {
1002 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
1003 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1004 if (snapshots != null) {
1005 LOG.error("Snapshots are present, but cleaners are not enabled.");
1006 checkSnapshotSupport();
1007 }
1008 }
1009 }
1010 }
1011
1012 @Override
1013 public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1014 IOException, UnsupportedOperationException {
1015 this.master = master;
1016
1017 this.rootDir = master.getMasterFileSystem().getRootDir();
1018 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1019
1020
1021 Configuration conf = master.getConfiguration();
1022 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1023 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1024 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1025
1026
1027 String name = master.getServerName().toString();
1028 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1029 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1030 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1031
1032 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1033 this.executorService = master.getExecutorService();
1034 resetTempDir();
1035 }
1036
1037 @Override
1038 public String getProcedureSignature() {
1039 return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1040 }
1041
1042 @Override
1043 public void execProcedure(ProcedureDescription desc) throws IOException {
1044 takeSnapshot(toSnapshotDescription(desc));
1045 }
1046
1047 @Override
1048 public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1049 return isSnapshotDone(toSnapshotDescription(desc));
1050 }
1051
1052 private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1053 throws IOException {
1054 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1055 if (!desc.hasInstance()) {
1056 throw new IOException("Snapshot name is not defined: " + desc.toString());
1057 }
1058 String snapshotName = desc.getInstance();
1059 List<NameStringPair> props = desc.getConfigurationList();
1060 String table = null;
1061 for (NameStringPair prop : props) {
1062 if ("table".equalsIgnoreCase(prop.getName())) {
1063 table = prop.getValue();
1064 }
1065 }
1066 if (table == null) {
1067 throw new IOException("Snapshot table is not defined: " + desc.toString());
1068 }
1069 TableName tableName = TableName.valueOf(table);
1070 builder.setTable(tableName.getNameAsString());
1071 builder.setName(snapshotName);
1072 builder.setType(SnapshotDescription.Type.FLUSH);
1073 return builder.build();
1074 }
1075 }