1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.Stoppable;
45 import org.apache.hadoop.hbase.catalog.MetaReader;
46 import org.apache.hadoop.hbase.errorhandling.ForeignException;
47 import org.apache.hadoop.hbase.executor.ExecutorService;
48 import org.apache.hadoop.hbase.master.AssignmentManager;
49 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
50 import org.apache.hadoop.hbase.master.MasterFileSystem;
51 import org.apache.hadoop.hbase.master.MasterServices;
52 import org.apache.hadoop.hbase.master.MetricsMaster;
53 import org.apache.hadoop.hbase.master.SnapshotSentinel;
54 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
55 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
56 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
57 import org.apache.hadoop.hbase.procedure.Procedure;
58 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
59 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
60 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
61 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
62 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
63 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
64 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
65 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
66 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
67 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
68 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
69 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
70 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
71 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
72 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
73 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
74 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
75 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
76 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
77 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
78 import org.apache.hadoop.hbase.util.FSUtils;
79 import org.apache.zookeeper.KeeperException;
80
81
82
83
84
85
86
87
88
89
90 @InterfaceAudience.Private
91 @InterfaceStability.Unstable
92 public class SnapshotManager extends MasterProcedureManager implements Stoppable {
93 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
94
95
96 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
97
98
99
100
101
102
103
104
105
106
107
108
109 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
110
111
112 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
113
114
115
116
117
118 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
119
120
121 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
122
123
124
125
126
127 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
128
129
130 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
131
132
133 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
134
135
136 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
137
138 private boolean stopped;
139 private MasterServices master;
140 private MetricsMaster metricsMaster;
141 private ProcedureCoordinator coordinator;
142
143
144 private boolean isSnapshotSupported = false;
145
146
147
148
149
150 private Map<TableName, SnapshotSentinel> snapshotHandlers =
151 new HashMap<TableName, SnapshotSentinel>();
152
153
154
155
156
157 private Map<TableName, SnapshotSentinel> restoreHandlers =
158 new HashMap<TableName, SnapshotSentinel>();
159
160 private Path rootDir;
161 private ExecutorService executorService;
162
163
164
165
166 private int snapshotLayoutVersion = SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_LATEST_FORMAT;
167
168 public SnapshotManager() {}
169
170
171
172
173
174
175
176 public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
177 ProcedureCoordinator coordinator, ExecutorService pool)
178 throws IOException, UnsupportedOperationException {
179 this.master = master;
180 this.metricsMaster = metricsMaster;
181
182 this.rootDir = master.getMasterFileSystem().getRootDir();
183 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
184
185 this.snapshotLayoutVersion = SnapshotDescriptionUtils.getDefaultSnapshotLayoutFormat(
186 master.getConfiguration());
187
188 this.coordinator = coordinator;
189 this.executorService = pool;
190 resetTempDir();
191 }
192
193
194
195
196
197
198 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
199 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
200 }
201
202
203
204
205
206
207
208 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
209 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
210
211 FileSystem fs = master.getMasterFileSystem().getFileSystem();
212 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
213
214
215 if (!fs.exists(snapshotDir)) {
216 return snapshotDescs;
217 }
218
219
220 FileStatus[] snapshots = fs.listStatus(snapshotDir,
221 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
222
223 for (FileStatus snapshot : snapshots) {
224 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
225
226 if (!fs.exists(info)) {
227 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
228 continue;
229 }
230 FSDataInputStream in = null;
231 try {
232 in = fs.open(info);
233 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
234 snapshotDescs.add(desc);
235 } catch (IOException e) {
236 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
237 } finally {
238 if (in != null) {
239 in.close();
240 }
241 }
242 }
243 return snapshotDescs;
244 }
245
246
247
248
249
250
251
252 void resetTempDir() throws IOException {
253
254 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
255 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
256 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
257 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
258 }
259 }
260 }
261
262
263
264
265
266
267
268 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
269
270
271 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
272 if (cpHost != null) {
273 cpHost.preDeleteSnapshot(snapshot);
274 }
275
276
277 if (!isSnapshotCompleted(snapshot)) {
278 throw new SnapshotDoesNotExistException(snapshot);
279 }
280
281 String snapshotName = snapshot.getName();
282 LOG.debug("Deleting snapshot: " + snapshotName);
283
284 MasterFileSystem fs = master.getMasterFileSystem();
285 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
286
287
288 if (!fs.getFileSystem().delete(snapshotDir, true)) {
289 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
290 }
291
292
293 if (cpHost != null) {
294 cpHost.postDeleteSnapshot(snapshot);
295 }
296
297 }
298
299
300
301
302
303
304
305
306
307 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
308
309 if (expected == null) {
310 throw new UnknownSnapshotException(
311 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
312 }
313
314 String ssString = ClientSnapshotDescriptionUtils.toString(expected);
315
316
317
318 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
319
320
321 cleanupSentinels();
322
323 if (handler == null) {
324
325
326
327
328
329
330 if (!isSnapshotCompleted(expected)) {
331 throw new UnknownSnapshotException("Snapshot " + ssString
332 + " is not currently running or one of the known completed snapshots.");
333 }
334
335 return true;
336 }
337
338
339 try {
340 handler.rethrowExceptionIfFailed();
341 } catch (ForeignException e) {
342
343 String status;
344 Procedure p = coordinator.getProcedure(expected.getName());
345 if (p != null) {
346 status = p.getStatus();
347 } else {
348 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
349 }
350 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
351 expected);
352 }
353
354
355 if (handler.isFinished()) {
356 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
357 return true;
358 } else if (LOG.isDebugEnabled()) {
359 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
360 }
361 return false;
362 }
363
364
365
366
367
368
369
370
371
372 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
373 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
374 if (isTakingSnapshot(snapshotTable)) {
375 return true;
376 }
377 Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
378 while (it.hasNext()) {
379 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
380 SnapshotSentinel sentinel = entry.getValue();
381 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
382 return true;
383 }
384 }
385 return false;
386 }
387
388
389
390
391
392
393
394 synchronized boolean isTakingSnapshot(final TableName tableName) {
395 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
396 return handler != null && !handler.isFinished();
397 }
398
399
400
401
402
403
404
405 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
406 throws HBaseSnapshotException {
407 FileSystem fs = master.getMasterFileSystem().getFileSystem();
408 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
409 TableName snapshotTable =
410 TableName.valueOf(snapshot.getTable());
411
412
413 if (isTakingSnapshot(snapshot)) {
414 SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
415 throw new SnapshotCreationException("Rejected taking "
416 + ClientSnapshotDescriptionUtils.toString(snapshot)
417 + " because we are already running another snapshot "
418 + (handler != null ? ("on the same table " +
419 ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
420 : "with the same name"), snapshot);
421 }
422
423
424 if (isRestoringTable(snapshotTable)) {
425 SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
426 throw new SnapshotCreationException("Rejected taking "
427 + ClientSnapshotDescriptionUtils.toString(snapshot)
428 + " because we are already have a restore in progress on the same snapshot "
429 + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
430 }
431
432 try {
433
434
435 fs.delete(workingDir, true);
436
437
438 if (!fs.mkdirs(workingDir)) {
439 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
440 + ") for snapshot" , snapshot);
441 }
442 } catch (HBaseSnapshotException e) {
443 throw e;
444 } catch (IOException e) {
445 throw new SnapshotCreationException(
446 "Exception while checking to see if snapshot could be started.", e, snapshot);
447 }
448 }
449
450
451
452
453
454
455 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
456 throws HBaseSnapshotException {
457
458 prepareToTakeSnapshot(snapshot);
459
460
461 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
462
463
464 DisabledTableSnapshotHandler handler =
465 new DisabledTableSnapshotHandler(snapshot, master);
466 snapshotTable(snapshot, handler);
467 }
468
469
470
471
472
473
474 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
475 throws HBaseSnapshotException {
476
477 prepareToTakeSnapshot(snapshot);
478
479
480 EnabledTableSnapshotHandler handler =
481 new EnabledTableSnapshotHandler(snapshot, master, this);
482 snapshotTable(snapshot, handler);
483 }
484
485
486
487
488
489
490
491
492
493 private synchronized void snapshotTable(SnapshotDescription snapshot,
494 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
495 try {
496 handler.prepare();
497 this.executorService.submit(handler);
498 this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
499 } catch (Exception e) {
500
501 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
502 try {
503 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
504 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
505 ClientSnapshotDescriptionUtils.toString(snapshot));
506 }
507 } catch (IOException e1) {
508 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
509 ClientSnapshotDescriptionUtils.toString(snapshot));
510 }
511
512 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
513 }
514 }
515
516
517
518
519
520
521
522
523 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
524
525 if (isSnapshotCompleted(snapshot)) {
526 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
527 + "' already stored on the filesystem.", snapshot);
528 }
529
530 LOG.debug("No existing snapshot, attempting snapshot...");
531
532
533 cleanupSentinels();
534
535
536 HTableDescriptor desc = null;
537 try {
538 desc = master.getTableDescriptors().get(
539 TableName.valueOf(snapshot.getTable()));
540 } catch (FileNotFoundException e) {
541 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
542 LOG.error(msg);
543 throw new SnapshotCreationException(msg, e, snapshot);
544 } catch (IOException e) {
545 throw new SnapshotCreationException("Error while geting table description for table "
546 + snapshot.getTable(), e, snapshot);
547 }
548 if (desc == null) {
549 throw new SnapshotCreationException("Table '" + snapshot.getTable()
550 + "' doesn't exist, can't take snapshot.", snapshot);
551 }
552
553
554 if (!snapshot.hasVersion()) {
555 snapshot = snapshot.toBuilder()
556 .setVersion(snapshotLayoutVersion)
557 .build();
558 }
559
560
561 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
562 if (cpHost != null) {
563 cpHost.preSnapshot(snapshot, desc);
564 }
565
566
567 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
568 AssignmentManager assignmentMgr = master.getAssignmentManager();
569 if (assignmentMgr.getZKTable().isEnabledTable(snapshotTable)) {
570 LOG.debug("Table enabled, starting distributed snapshot.");
571 snapshotEnabledTable(snapshot);
572 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
573 }
574
575 else if (assignmentMgr.getZKTable().isDisabledTable(snapshotTable)) {
576 LOG.debug("Table is disabled, running snapshot entirely on master.");
577 snapshotDisabledTable(snapshot);
578 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
579 } else {
580 LOG.error("Can't snapshot table '" + snapshot.getTable()
581 + "', isn't open or closed, we don't know what to do!");
582 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
583 + " isn't fully open.");
584 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
585 }
586
587
588 if (cpHost != null) {
589 cpHost.postSnapshot(snapshot, desc);
590 }
591 }
592
593
594
595
596
597
598
599
600
601
602 public synchronized void setSnapshotHandlerForTesting(
603 final TableName tableName,
604 final SnapshotSentinel handler) {
605 if (handler != null) {
606 this.snapshotHandlers.put(tableName, handler);
607 } else {
608 this.snapshotHandlers.remove(tableName);
609 }
610 }
611
612
613
614
615 ProcedureCoordinator getCoordinator() {
616 return coordinator;
617 }
618
619
620
621
622
623
624
625
626
627
628
629 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
630 try {
631 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
632 FileSystem fs = master.getMasterFileSystem().getFileSystem();
633
634 return fs.exists(snapshotDir);
635 } catch (IllegalArgumentException iae) {
636 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
637 }
638 }
639
640
641
642
643
644
645
646
647 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
648 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
649 TableName tableName = hTableDescriptor.getTableName();
650
651
652 if (isTakingSnapshot(tableName)) {
653 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
654 }
655
656
657 if (isRestoringTable(tableName)) {
658 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
659 }
660
661 try {
662 CloneSnapshotHandler handler =
663 new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
664 this.executorService.submit(handler);
665 this.restoreHandlers.put(tableName, handler);
666 } catch (Exception e) {
667 String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
668 " on table=" + tableName;
669 LOG.error(msg, e);
670 throw new RestoreSnapshotException(msg, e);
671 }
672 }
673
674
675
676
677
678
679 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
680 FileSystem fs = master.getMasterFileSystem().getFileSystem();
681 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
682 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
683
684
685 if (!fs.exists(snapshotDir)) {
686 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
687 throw new SnapshotDoesNotExistException(reqSnapshot);
688 }
689
690
691 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
692 SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
693 snapshotDir, fsSnapshot);
694 HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
695 TableName tableName = TableName.valueOf(reqSnapshot.getTable());
696
697
698 cleanupSentinels();
699
700
701 SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
702
703
704 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
705 if (master.getAssignmentManager().getZKTable().isEnabledTable(
706 TableName.valueOf(fsSnapshot.getTable()))) {
707 throw new UnsupportedOperationException("Table '" +
708 TableName.valueOf(fsSnapshot.getTable()) + "' must be disabled in order to " +
709 "perform a restore operation" +
710 ".");
711 }
712
713
714 if (cpHost != null) {
715 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
716 }
717 restoreSnapshot(fsSnapshot, snapshotTableDesc);
718 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
719
720 if (cpHost != null) {
721 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
722 }
723 } else {
724 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
725 if (cpHost != null) {
726 cpHost.preCloneSnapshot(reqSnapshot, htd);
727 }
728 cloneSnapshot(fsSnapshot, htd);
729 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
730
731 if (cpHost != null) {
732 cpHost.postCloneSnapshot(reqSnapshot, htd);
733 }
734 }
735 }
736
737
738
739
740
741
742
743
744 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
745 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
746 TableName tableName = hTableDescriptor.getTableName();
747
748
749 if (isTakingSnapshot(tableName)) {
750 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
751 }
752
753
754 if (isRestoringTable(tableName)) {
755 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
756 }
757
758 try {
759 RestoreSnapshotHandler handler =
760 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
761 this.executorService.submit(handler);
762 restoreHandlers.put(tableName, handler);
763 } catch (Exception e) {
764 String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
765 snapshot) +
766 " on table=" + tableName;
767 LOG.error(msg, e);
768 throw new RestoreSnapshotException(msg, e);
769 }
770 }
771
772
773
774
775
776
777
778 private synchronized boolean isRestoringTable(final TableName tableName) {
779 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
780 return(sentinel != null && !sentinel.isFinished());
781 }
782
783
784
785
786
787
788
789
790
791 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
792
793
794 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
795
796
797 cleanupSentinels();
798
799 if (sentinel == null) {
800
801 return true;
802 }
803
804 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
805 + sentinel.getSnapshot().getName() + " table=" +
806 TableName.valueOf(snapshot.getTable()));
807
808
809 sentinel.rethrowExceptionIfFailed();
810
811
812 if (sentinel.isFinished()) {
813 LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
814 " has completed. Notifying the client.");
815 return true;
816 }
817
818 if (LOG.isDebugEnabled()) {
819 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
820 ClientSnapshotDescriptionUtils.toString(snapshot));
821 }
822 return false;
823 }
824
825
826
827
828
829
830
831
832 private synchronized SnapshotSentinel removeSentinelIfFinished(
833 final Map<TableName, SnapshotSentinel> sentinels,
834 final SnapshotDescription snapshot) {
835 if (!snapshot.hasTable()) {
836 return null;
837 }
838
839 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
840 SnapshotSentinel h = sentinels.get(snapshotTable);
841 if (h == null) {
842 return null;
843 }
844
845 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
846
847 return null;
848 }
849
850
851 if (h.isFinished()) {
852 sentinels.remove(snapshotTable);
853 }
854
855 return h;
856 }
857
858
859
860
861
862
863
864
865 private void cleanupSentinels() {
866 cleanupSentinels(this.snapshotHandlers);
867 cleanupSentinels(this.restoreHandlers);
868 }
869
870
871
872
873
874
875 private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
876 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
877 Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
878 sentinels.entrySet().iterator();
879 while (it.hasNext()) {
880 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
881 SnapshotSentinel sentinel = entry.getValue();
882 if (sentinel.isFinished() &&
883 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
884 {
885 it.remove();
886 }
887 }
888 }
889
890
891
892
893
894 @Override
895 public void stop(String why) {
896
897 if (this.stopped) return;
898
899 this.stopped = true;
900
901 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
902 snapshotHandler.cancel(why);
903 }
904
905
906 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
907 restoreHandler.cancel(why);
908 }
909 try {
910 coordinator.close();
911 } catch (IOException e) {
912 LOG.error("stop ProcedureCoordinator error", e);
913 }
914 }
915
916 @Override
917 public boolean isStopped() {
918 return this.stopped;
919 }
920
921
922
923
924
925
926 public void checkSnapshotSupport() throws UnsupportedOperationException {
927 if (!this.isSnapshotSupported) {
928 throw new UnsupportedOperationException(
929 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
930 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
931 }
932 }
933
934
935
936
937
938
939
940
941
942
943
944 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
945 throws IOException, UnsupportedOperationException {
946
947 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
948 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
949 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
950
951
952 Set<String> hfileCleaners = new HashSet<String>();
953 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
954 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
955
956 Set<String> logCleaners = new HashSet<String>();
957 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
958 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
959
960
961 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
962 FileSystem fs = mfs.getFileSystem();
963 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
964 if (ss != null && !ss.isEmpty()) {
965 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
966 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
967 }
968
969
970
971
972 if (snapshotEnabled) {
973
974 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
975 hfileCleaners.add(HFileLinkCleaner.class.getName());
976 logCleaners.add(SnapshotLogCleaner.class.getName());
977
978
979 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
980 hfileCleaners.toArray(new String[hfileCleaners.size()]));
981 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
982 logCleaners.toArray(new String[logCleaners.size()]));
983 } else {
984
985 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
986 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
987 hfileCleaners.contains(HFileLinkCleaner.class.getName());
988
989
990 if (snapshotEnabled) {
991 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
992 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
993 (userDisabled ? "is set to 'false'." : "is not set."));
994 }
995 }
996
997
998 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
999
1000
1001
1002 if (!snapshotEnabled) {
1003 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
1004 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
1005 if (fs.exists(snapshotDir)) {
1006 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
1007 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1008 if (snapshots != null) {
1009 LOG.error("Snapshots are present, but cleaners are not enabled.");
1010 checkSnapshotSupport();
1011 }
1012 }
1013 }
1014 }
1015
1016 @Override
1017 public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1018 IOException, UnsupportedOperationException {
1019 this.master = master;
1020 this.metricsMaster = metricsMaster;
1021
1022 this.rootDir = master.getMasterFileSystem().getRootDir();
1023 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1024
1025
1026 Configuration conf = master.getConfiguration();
1027 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1028 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1029 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1030
1031
1032 String name = master.getServerName().toString();
1033 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1034 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1035 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1036
1037 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1038 this.executorService = master.getExecutorService();
1039 resetTempDir();
1040 }
1041
1042 @Override
1043 public String getProcedureSignature() {
1044 return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1045 }
1046
1047 @Override
1048 public void execProcedure(ProcedureDescription desc) throws IOException {
1049 takeSnapshot(toSnapshotDescription(desc));
1050 }
1051
1052 @Override
1053 public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1054 return isSnapshotDone(toSnapshotDescription(desc));
1055 }
1056
1057 private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1058 throws IOException {
1059 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1060 if (!desc.hasInstance()) {
1061 throw new IOException("Snapshot name is not defined: " + desc.toString());
1062 }
1063 String snapshotName = desc.getInstance();
1064 List<NameStringPair> props = desc.getConfigurationList();
1065 String table = null;
1066 for (NameStringPair prop : props) {
1067 if ("table".equalsIgnoreCase(prop.getName())) {
1068 table = prop.getValue();
1069 }
1070 }
1071 if (table == null) {
1072 throw new IOException("Snapshot table is not defined: " + desc.toString());
1073 }
1074 TableName tableName = TableName.valueOf(table);
1075 builder.setTable(tableName.getNameAsString());
1076 builder.setName(snapshotName);
1077 builder.setType(SnapshotDescription.Type.FLUSH);
1078 return builder.build();
1079 }
1080 }