1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.fs.Path;
25 import org.apache.hadoop.hbase.HRegionInfo;
26 import org.apache.hadoop.hbase.HTableDescriptor;
27 import org.apache.hadoop.hbase.MetaTableAccessor;
28 import org.apache.hadoop.hbase.NotServingRegionException;
29 import org.apache.hadoop.hbase.ServerName;
30 import org.apache.hadoop.hbase.TableName;
31 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.client.Admin;
34 import org.apache.hadoop.hbase.client.Connection;
35 import org.apache.hadoop.hbase.client.ConnectionFactory;
36 import org.apache.hadoop.hbase.client.HConnection;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Table;
39 import org.apache.hadoop.hbase.master.RegionState;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
42 import org.apache.hadoop.hbase.regionserver.HRegion;
43 import org.apache.zookeeper.KeeperException;
44
45 import java.io.IOException;
46 import java.util.Collection;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Random;
50
51
52
53
54
55 @InterfaceAudience.Private
56 public class HBaseFsckRepair {
57 public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
58
59
60
61
62
63
64
65
66
67
68 public static void fixMultiAssignment(HConnection connection, HRegionInfo region,
69 List<ServerName> servers)
70 throws IOException, KeeperException, InterruptedException {
71 HRegionInfo actualRegion = new HRegionInfo(region);
72
73
74 for(ServerName server : servers) {
75 closeRegionSilentlyAndWait(connection, server, actualRegion);
76 }
77
78
79 forceOfflineInZK(connection.getAdmin(), actualRegion);
80 }
81
82
83
84
85
86
87
88
89
90
91
92
93
94 public static void fixUnassigned(Admin admin, HRegionInfo region)
95 throws IOException, KeeperException, InterruptedException {
96 HRegionInfo actualRegion = new HRegionInfo(region);
97
98
99 forceOfflineInZK(admin, actualRegion);
100 }
101
102
103
104
105
106
107
108
109
110
111
112
113
114 private static void forceOfflineInZK(Admin admin, final HRegionInfo region)
115 throws ZooKeeperConnectionException, KeeperException, IOException, InterruptedException {
116 admin.assign(region.getRegionName());
117 }
118
119
120
121
122 public static void waitUntilAssigned(Admin admin,
123 HRegionInfo region) throws IOException, InterruptedException {
124 long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
125 long expiration = timeout + System.currentTimeMillis();
126 while (System.currentTimeMillis() < expiration) {
127 try {
128 Map<String, RegionState> rits=
129 admin.getClusterStatus().getRegionsInTransition();
130
131 if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
132
133 return;
134 }
135
136 LOG.info("Region still in transition, waiting for "
137 + "it to become assigned: " + region);
138 } catch (IOException e) {
139 LOG.warn("Exception when waiting for region to become assigned,"
140 + " retrying", e);
141 }
142 Thread.sleep(1000);
143 }
144 throw new IOException("Region " + region + " failed to move out of " +
145 "transition within timeout " + timeout + "ms");
146 }
147
148
149
150
151
152 @SuppressWarnings("deprecation")
153 public static void closeRegionSilentlyAndWait(HConnection connection,
154 ServerName server, HRegionInfo region) throws IOException, InterruptedException {
155 AdminService.BlockingInterface rs = connection.getAdmin(server);
156 try {
157 ProtobufUtil.closeRegion(rs, server, region.getRegionName(), false);
158 } catch (IOException e) {
159 LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
160 }
161 long timeout = connection.getConfiguration()
162 .getLong("hbase.hbck.close.timeout", 120000);
163 long expiration = timeout + System.currentTimeMillis();
164 while (System.currentTimeMillis() < expiration) {
165 try {
166 HRegionInfo rsRegion =
167 ProtobufUtil.getRegionInfo(rs, region.getRegionName());
168 if (rsRegion == null) return;
169 } catch (IOException ioe) {
170 if (ioe instanceof NotServingRegionException)
171 return;
172 LOG.warn("Exception when retrieving regioninfo from: " + region.getRegionNameAsString(), ioe);
173 }
174 Thread.sleep(1000);
175 }
176 throw new IOException("Region " + region + " failed to close within"
177 + " timeout " + timeout);
178 }
179
180
181
182
183 public static void fixMetaHoleOnline(Configuration conf,
184 HRegionInfo hri) throws IOException {
185 Connection conn = ConnectionFactory.createConnection(conf);
186 Table meta = conn.getTable(TableName.META_TABLE_NAME);
187 MetaTableAccessor.addRegionToMeta(meta, hri);
188 meta.close();
189 conn.close();
190 }
191
192
193
194
195 public static HRegion createHDFSRegionDir(Configuration conf,
196 HRegionInfo hri, HTableDescriptor htd) throws IOException {
197
198 Path root = FSUtils.getRootDir(conf);
199 HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
200
201
202 HRegion.closeHRegion(region);
203 return region;
204 }
205 }