1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.migration;
19
20 import java.io.IOException;
21 import java.util.List;
22
23 import org.apache.commons.cli.CommandLine;
24 import org.apache.commons.cli.CommandLineParser;
25 import org.apache.commons.cli.GnuParser;
26 import org.apache.commons.cli.HelpFormatter;
27 import org.apache.commons.cli.Option;
28 import org.apache.commons.cli.Options;
29 import org.apache.commons.cli.ParseException;
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configured;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.Abortable;
37 import org.apache.hadoop.hbase.HBaseConfiguration;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.FSUtils;
42 import org.apache.hadoop.hbase.util.HFileV1Detector;
43 import org.apache.hadoop.hbase.util.ZKDataMigrator;
44 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
45 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46 import org.apache.hadoop.util.Tool;
47 import org.apache.hadoop.util.ToolRunner;
48
49 public class UpgradeTo96 extends Configured implements Tool {
50 private static final Log LOG = LogFactory.getLog(UpgradeTo96.class);
51
52 private Options options = new Options();
53
54
55
56 private boolean upgrade;
57
58
59
60 private boolean checkForHFileV1;
61
62
63
64 private String dirToCheckForHFileV1;
65
66 UpgradeTo96() {
67 setOptions();
68 }
69
70 private void setOptions() {
71 options.addOption("h", "help", false, "Help");
72 options.addOption(new Option("check", false, "Run upgrade check; looks for HFileV1 "
73 + " under ${hbase.rootdir} or provided 'dir' directory."));
74 options.addOption(new Option("execute", false, "Run upgrade; zk and hdfs must be up, hbase down"));
75 Option pathOption = new Option("dir", true,
76 "Relative path of dir to check for HFileV1s.");
77 pathOption.setRequired(false);
78 options.addOption(pathOption);
79 }
80
81 private boolean parseOption(String[] args) throws ParseException {
82 if (args.length == 0) return false;
83
84 CommandLineParser parser = new GnuParser();
85 CommandLine cmd = parser.parse(options, args);
86 if (cmd.hasOption("h")) {
87 printUsage();
88 return false;
89 }
90 if (cmd.hasOption("execute")) upgrade = true;
91 if (cmd.hasOption("check")) checkForHFileV1 = true;
92 if (checkForHFileV1 && cmd.hasOption("dir")) {
93 this.dirToCheckForHFileV1 = cmd.getOptionValue("dir");
94 }
95 return true;
96 }
97
98 private void printUsage() {
99 HelpFormatter formatter = new HelpFormatter();
100 formatter.printHelp("$bin/hbase upgrade -check [-dir DIR]|-execute", options);
101 System.out.println("Read http://hbase.apache.org/book.html#upgrade0.96 before attempting upgrade");
102 System.out.println();
103 System.out.println("Example usage:");
104 System.out.println();
105 System.out.println("Run upgrade check; looks for HFileV1s under ${hbase.rootdir}:");
106 System.out.println(" $ bin/hbase upgrade -check");
107 System.out.println();
108 System.out.println("Run the upgrade: ");
109 System.out.println(" $ bin/hbase upgrade -execute");
110 }
111
112 @Override
113 public int run(String[] args) throws Exception {
114 if (!parseOption(args)) {
115 printUsage();
116 return -1;
117 }
118 if (checkForHFileV1) {
119 int res = doHFileV1Check();
120 if (res == 0) LOG.info("No HFileV1 found.");
121 else {
122 LOG.warn("There are some HFileV1, or corrupt files (files with incorrect major version).");
123 }
124 return res;
125 }
126
127
128 else if (upgrade) {
129 if (isAnyHBaseProcessAlive()) {
130 LOG.error("Some HBase processes are still alive, or znodes not expired yet. "
131 + "Please stop them before upgrade or try after some time.");
132 throw new IOException("Some HBase processes are still alive, or znodes not expired yet");
133 }
134 return executeUpgrade();
135 }
136 return -1;
137 }
138
139 private boolean isAnyHBaseProcessAlive() throws IOException {
140 ZooKeeperWatcher zkw = null;
141 try {
142 zkw = new ZooKeeperWatcher(getConf(), "Check Live Processes.", new Abortable() {
143 private boolean aborted = false;
144
145 @Override
146 public void abort(String why, Throwable e) {
147 LOG.warn("Got aborted with reason: " + why + ", and error: " + e);
148 this.aborted = true;
149 }
150
151 @Override
152 public boolean isAborted() {
153 return this.aborted;
154 }
155
156 });
157 boolean liveProcessesExists = false;
158 if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
159 return false;
160 }
161 if (ZKUtil.checkExists(zkw, zkw.backupMasterAddressesZNode) != -1) {
162 List<String> backupMasters = ZKUtil
163 .listChildrenNoWatch(zkw, zkw.backupMasterAddressesZNode);
164 if (!backupMasters.isEmpty()) {
165 LOG.warn("Backup master(s) " + backupMasters
166 + " are alive or backup-master znodes not expired.");
167 liveProcessesExists = true;
168 }
169 }
170 if (ZKUtil.checkExists(zkw, zkw.rsZNode) != -1) {
171 List<String> regionServers = ZKUtil.listChildrenNoWatch(zkw, zkw.rsZNode);
172 if (!regionServers.isEmpty()) {
173 LOG.warn("Region server(s) " + regionServers + " are alive or rs znodes not expired.");
174 liveProcessesExists = true;
175 }
176 }
177 if (ZKUtil.checkExists(zkw, zkw.getMasterAddressZNode()) != -1) {
178 byte[] data = ZKUtil.getData(zkw, zkw.getMasterAddressZNode());
179 if (data != null && !Bytes.equals(data, HConstants.EMPTY_BYTE_ARRAY)) {
180 LOG.warn("Active master at address " + Bytes.toString(data)
181 + " is still alive or master znode not expired.");
182 liveProcessesExists = true;
183 }
184 }
185 return liveProcessesExists;
186 } catch (Exception e) {
187 LOG.error("Got exception while checking live hbase processes", e);
188 throw new IOException(e);
189 } finally {
190 if (zkw != null) {
191 zkw.close();
192 }
193 }
194 }
195
196 private int doHFileV1Check() throws Exception {
197 String[] args = null;
198 if (dirToCheckForHFileV1 != null) args = new String[] { "-p" + dirToCheckForHFileV1 };
199 return ToolRunner.run(getConf(), new HFileV1Detector(), args);
200 }
201
202
203
204
205
206
207
208
209
210
211 private int executeUpgrade() throws Exception {
212 executeTool("Namespace upgrade", new NamespaceUpgrade(),
213 new String[] { "--upgrade" }, 0);
214 executeTool("Znode upgrade", new ZKDataMigrator(), null, 0);
215 doOfflineLogSplitting();
216 return 0;
217 }
218
219 private void executeTool(String toolMessage, Tool tool, String[] args, int expectedResult)
220 throws Exception {
221 LOG.info("Starting " + toolMessage);
222 int res = ToolRunner.run(getConf(), tool, new String[] { "--upgrade" });
223 if (res != expectedResult) {
224 LOG.error(toolMessage + "returned " + res + ", expected " + expectedResult);
225 throw new Exception("Unexpected return code from " + toolMessage);
226 }
227 LOG.info("Successfully completed " + toolMessage);
228 }
229
230
231
232
233
234 private void doOfflineLogSplitting() throws Exception {
235 LOG.info("Starting Log splitting");
236 final Path rootDir = FSUtils.getRootDir(getConf());
237 final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
238 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
239 Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
240 FileStatus[] regionServerLogDirs = FSUtils.listStatus(fs, logDir);
241 if (regionServerLogDirs == null || regionServerLogDirs.length == 0) {
242 LOG.info("No log directories to split, returning");
243 return;
244 }
245 try {
246 for (FileStatus regionServerLogDir : regionServerLogDirs) {
247
248 HLogSplitter.split(rootDir, regionServerLogDir.getPath(), oldLogDir, fs, getConf());
249 }
250 LOG.info("Successfully completed Log splitting");
251 } catch (Exception e) {
252 LOG.error("Got exception while doing Log splitting ", e);
253 throw e;
254 }
255 }
256
257 public static void main(String[] args) throws Exception {
258 System.exit(ToolRunner.run(HBaseConfiguration.create(), new UpgradeTo96(), args));
259 }
260 }