View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.util.Map;
24  import java.util.TreeMap;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.io.HLogLink;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.regionserver.wal.HLog;
35  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
36  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
37  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
38  import org.apache.hadoop.hbase.util.Bytes;
39  
40  /**
41   * If the snapshot has references to one or more log files,
42   * those must be split (each log contains multiple tables and regions)
43   * and must be placed in the region/recovered.edits folder.
44   * (recovered.edits files will be played on region startup)
45   *
46   * In case of Restore: the log can just be split in the recovered.edits folder.
47   * In case of Clone: each entry in the log must be modified to use the new region name.
48   * (region names are encoded with: tableName, startKey, regionIdTimeStamp)
49   *
50   * We can't use the normal split code, because the HLogKey contains the
51   * table name and the region name, and in case of "clone from snapshot"
52   * region name and table name will be different and must be replaced in
53   * the recovered.edits.
54   */
55  @InterfaceAudience.Private
56  class SnapshotLogSplitter implements Closeable {
57    static final Log LOG = LogFactory.getLog(SnapshotLogSplitter.class);
58  
59    private final class LogWriter implements Closeable {
60      private HLog.Writer writer;
61      private Path logFile;
62      private long seqId;
63  
64      public LogWriter(final Configuration conf, final FileSystem fs,
65          final Path logDir, long seqId) throws IOException {
66        logFile = new Path(logDir, logFileName(seqId, true));
67        this.writer = HLogFactory.createWriter(fs, logFile, conf);
68        this.seqId = seqId;
69      }
70  
71      public void close() throws IOException {
72        writer.close();
73  
74        Path finalFile = new Path(logFile.getParent(), logFileName(seqId, false));
75        LOG.debug("LogWriter tmpLogFile=" + logFile + " -> logFile=" + finalFile);
76        fs.rename(logFile, finalFile);
77      }
78  
79      public void append(final HLog.Entry entry) throws IOException {
80        writer.append(entry);
81        if (seqId < entry.getKey().getLogSeqNum()) {
82          seqId = entry.getKey().getLogSeqNum();
83        }
84      }
85  
86      private String logFileName(long seqId, boolean temp) {
87        String fileName = String.format("%019d", seqId);
88        if (temp) fileName += HLog.RECOVERED_LOG_TMPFILE_SUFFIX;
89        return fileName;
90      }
91    }
92  
93    private final Map<byte[], LogWriter> regionLogWriters =
94        new TreeMap<byte[], LogWriter>(Bytes.BYTES_COMPARATOR);
95  
96    private final Map<byte[], byte[]> regionsMap;
97    private final Configuration conf;
98    private final byte[] snapshotTableName;
99    private final byte[] tableName;
100   private final Path tableDir;
101   private final FileSystem fs;
102 
103   /**
104    * @params tableName snapshot table name
105    * @params regionsMap maps original region names to the new ones.
106    */
107   public SnapshotLogSplitter(final Configuration conf, final FileSystem fs,
108       final Path tableDir, final byte[] snapshotTableName,
109       final Map<byte[], byte[]> regionsMap) {
110     this.regionsMap = regionsMap;
111     this.snapshotTableName = snapshotTableName;
112     this.tableName = Bytes.toBytes(tableDir.getName());
113     this.tableDir = tableDir;
114     this.conf = conf;
115     this.fs = fs;
116   }
117 
118   public void close() throws IOException {
119     for (LogWriter writer: regionLogWriters.values()) {
120       writer.close();
121     }
122   }
123 
124   public void splitLog(final String serverName, final String logfile) throws IOException {
125     LOG.debug("Restore log=" + logfile + " server=" + serverName +
126               " for snapshotTable=" + Bytes.toString(snapshotTableName) +
127               " to table=" + Bytes.toString(tableName));
128     splitLog(new HLogLink(conf, serverName, logfile).getAvailablePath(fs));
129   }
130 
131   public void splitRecoveredEdit(final Path editPath) throws IOException {
132     LOG.debug("Restore recover.edits=" + editPath +
133               " for snapshotTable=" + Bytes.toString(snapshotTableName) +
134               " to table=" + Bytes.toString(tableName));
135     splitLog(editPath);
136   }
137 
138   /**
139    * Split the snapshot HLog reference into regions recovered.edits.
140    *
141    * The HLogKey contains the table name and the region name,
142    * and they must be changed to the restored table names.
143    *
144    * @param logPath Snapshot HLog reference path
145    */
146   public void splitLog(final Path logPath) throws IOException {
147     HLog.Reader log = HLogFactory.createReader(fs, logPath, conf);
148     try {
149       HLog.Entry entry;
150       LogWriter writer = null;
151       byte[] regionName = null;
152       byte[] newRegionName = null;
153       while ((entry = log.next()) != null) {
154         HLogKey key = entry.getKey();
155 
156         // We're interested only in the snapshot table that we're restoring
157         if (!Bytes.equals(key.getTablename(), snapshotTableName)) continue;
158 
159         // Writer for region.
160         if (!Bytes.equals(regionName, key.getEncodedRegionName())) {
161           regionName = key.getEncodedRegionName().clone();
162 
163           // Get the new region name in case of clone, or use the original one
164           newRegionName = regionsMap.get(regionName);
165           if (newRegionName == null) newRegionName = regionName;
166 
167           writer = getOrCreateWriter(newRegionName, key.getLogSeqNum());
168           LOG.debug("+ regionName=" + Bytes.toString(regionName));
169         }
170 
171         // Append Entry
172         key = new HLogKey(newRegionName, tableName,
173                           key.getLogSeqNum(), key.getWriteTime(), key.getClusterId());
174         writer.append(new HLog.Entry(key, entry.getEdit()));
175       }
176     } catch (IOException e) {
177       LOG.warn("Something wrong during the log split", e);
178     } finally {
179       log.close();
180     }
181   }
182 
183   /**
184    * Create a LogWriter for specified region if not already created.
185    */
186   private LogWriter getOrCreateWriter(final byte[] regionName, long seqId) throws IOException {
187     LogWriter writer = regionLogWriters.get(regionName);
188     if (writer == null) {
189       Path regionDir = HRegion.getRegionDir(tableDir, Bytes.toString(regionName));
190       Path dir = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
191       fs.mkdirs(dir);
192 
193       writer = new LogWriter(conf, fs, dir, seqId);
194       regionLogWriters.put(regionName, writer);
195     }
196     return(writer);
197   }
198 }