View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import com.google.common.cache.CacheBuilder;
21  import com.google.common.cache.CacheLoader;
22  import com.google.common.cache.LoadingCache;
23  import com.google.common.collect.Lists;
24  import com.google.common.util.concurrent.ListenableFuture;
25  import com.google.common.util.concurrent.ListeningExecutorService;
26  import com.google.common.util.concurrent.MoreExecutors;
27  import com.google.common.util.concurrent.ThreadFactoryBuilder;
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.ClusterStatus;
32  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.master.AssignmentManager;
39  import org.apache.hadoop.hbase.master.MasterServices;
40  import org.apache.hadoop.hbase.master.RegionStates;
41  import org.apache.hadoop.hbase.regionserver.HRegion;
42  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
43  
44  import java.io.FileNotFoundException;
45  import java.io.IOException;
46  import java.util.ArrayList;
47  import java.util.Collection;
48  import java.util.HashMap;
49  import java.util.List;
50  import java.util.Set;
51  import java.util.concurrent.Callable;
52  import java.util.concurrent.ExecutionException;
53  import java.util.concurrent.Executors;
54  import java.util.concurrent.TimeUnit;
55  
56  /**
57   * This will find where data for a region is located in HDFS. It ranks
58   * {@link ServerName}'s by the size of the store files they are holding for a
59   * given region.
60   *
61   */
62  @InterfaceAudience.Private
63  class RegionLocationFinder {
64    private static final Log LOG = LogFactory.getLog(RegionLocationFinder.class);
65    private static final long DEFAULT_CACHE_TIME = 30 * 60 * 1000; // 30 minutes
66    private Configuration conf;
67    private volatile ClusterStatus status;
68    private MasterServices services;
69    private final ListeningExecutorService executor;
70    private long cacheTime = DEFAULT_CACHE_TIME;
71    private long lastFullRefresh = 0;
72  
73    private CacheLoader<HRegionInfo, HDFSBlocksDistribution> loader =
74        new CacheLoader<HRegionInfo, HDFSBlocksDistribution>() {
75  
76          public ListenableFuture<HDFSBlocksDistribution> reload(final HRegionInfo hri,
77                 HDFSBlocksDistribution oldValue) throws Exception {
78            return executor.submit(new Callable<HDFSBlocksDistribution>() {
79              @Override
80              public HDFSBlocksDistribution call() throws Exception {
81                return internalGetTopBlockLocation(hri);
82              }
83            });
84          }
85  
86          @Override
87          public HDFSBlocksDistribution load(HRegionInfo key) throws Exception {
88            return internalGetTopBlockLocation(key);
89          }
90        };
91  
92    // The cache for where regions are located.
93    private LoadingCache<HRegionInfo, HDFSBlocksDistribution> cache = null;
94  
95    RegionLocationFinder() {
96      this.cache = createCache();
97      executor = MoreExecutors.listeningDecorator(
98          Executors.newScheduledThreadPool(
99              5,
100             new ThreadFactoryBuilder().
101                 setDaemon(true)
102                 .setNameFormat("region-location-%d")
103                 .build()));
104   }
105 
106   /**
107    * Create a cache for region to list of servers
108    * @param time time to cache the locations
109    * @return A new Cache.
110    */
111   private LoadingCache<HRegionInfo, HDFSBlocksDistribution> createCache() {
112     return CacheBuilder.newBuilder()
113         .expireAfterWrite(cacheTime, TimeUnit.MILLISECONDS)
114         .build(loader);
115   }
116 
117   public Configuration getConf() {
118     return conf;
119   }
120 
121   public void setConf(Configuration conf) {
122     this.conf = conf;
123     // Preserve this tunable in 0.98
124     this.cacheTime = TimeUnit.SECONDS.toMillis(
125       conf.getInt("hbase.master.balancer.regionLocationCacheTime", 30));
126   }
127 
128   public void setServices(MasterServices services) {
129     this.services = services;
130   }
131 
132   public void setClusterStatus(ClusterStatus status) {
133     long currentTime = EnvironmentEdgeManager.currentTimeMillis();
134     this.status = status;
135     if (currentTime > lastFullRefresh + (cacheTime / 2)) {
136       // Only count the refresh if it includes user tables ( eg more than meta and namespace ).
137       lastFullRefresh = scheduleFullRefresh()?currentTime:lastFullRefresh;
138     }
139 
140   }
141 
142   /**
143    * Refresh all the region locations.
144    *
145    * @return true if user created regions got refreshed.
146    */
147   private boolean scheduleFullRefresh() {
148     // Protect from anything being null while starting up.
149     if (services == null) {
150       return false;
151     }
152     AssignmentManager am = services.getAssignmentManager();
153 
154     if (am == null) {
155       return false;
156     }
157     RegionStates regionStates = am.getRegionStates();
158     if (regionStates == null) {
159       return false;
160     }
161 
162     Set<HRegionInfo> regions = regionStates.getRegionAssignments().keySet();
163     boolean includesUserTables = false;
164     for (final HRegionInfo hri : regions) {
165       cache.refresh(hri);
166       includesUserTables = includesUserTables || !hri.getTable().isSystemTable();
167     }
168     return includesUserTables;
169   }
170 
171   protected List<ServerName> getTopBlockLocations(HRegionInfo region) {
172     HDFSBlocksDistribution blocksDistribution = getBlockDistribution(region);
173     List<String> topHosts = blocksDistribution.getTopHosts();
174     return mapHostNameToServerName(topHosts);
175   }
176 
177   /**
178    * Returns an ordered list of hosts that are hosting the blocks for this
179    * region. The weight of each host is the sum of the block lengths of all
180    * files on that host, so the first host in the list is the server which holds
181    * the most bytes of the given region's HFiles.
182    *
183    * @param region region
184    * @return ordered list of hosts holding blocks of the specified region
185    */
186   protected HDFSBlocksDistribution internalGetTopBlockLocation(HRegionInfo region) {
187     try {
188       HTableDescriptor tableDescriptor = getTableDescriptor(region.getTable());
189       if (tableDescriptor != null) {
190         HDFSBlocksDistribution blocksDistribution =
191             HRegion.computeHDFSBlocksDistribution(getConf(), tableDescriptor, region);
192         return blocksDistribution;
193       }
194     } catch (IOException ioe) {
195       LOG.warn("IOException during HDFSBlocksDistribution computation. for " + "region = "
196           + region.getEncodedName(), ioe);
197     }
198 
199     return new HDFSBlocksDistribution();
200   }
201 
202   /**
203    * return HTableDescriptor for a given tableName
204    *
205    * @param tableName the table name
206    * @return HTableDescriptor
207    * @throws IOException
208    */
209   protected HTableDescriptor getTableDescriptor(TableName tableName) throws IOException {
210     HTableDescriptor tableDescriptor = null;
211     try {
212       if (this.services != null && this.services.getTableDescriptors() != null) {
213         tableDescriptor = this.services.getTableDescriptors().get(tableName);
214       }
215     } catch (FileNotFoundException fnfe) {
216       LOG.debug("FileNotFoundException during getTableDescriptors." + " Current table name = "
217           + tableName, fnfe);
218     }
219 
220     return tableDescriptor;
221   }
222 
223   /**
224    * Map hostname to ServerName, The output ServerName list will have the same
225    * order as input hosts.
226    *
227    * @param hosts the list of hosts
228    * @return ServerName list
229    */
230   protected List<ServerName> mapHostNameToServerName(List<String> hosts) {
231     if (hosts == null || status == null) {
232       if (hosts == null) {
233         LOG.warn("RegionLocationFinder top hosts is null");
234       }
235       return Lists.newArrayList();
236     }
237 
238     List<ServerName> topServerNames = new ArrayList<ServerName>();
239     Collection<ServerName> regionServers = status.getServers();
240 
241     // create a mapping from hostname to ServerName for fast lookup
242     HashMap<String, ServerName> hostToServerName = new HashMap<String, ServerName>();
243     for (ServerName sn : regionServers) {
244       hostToServerName.put(sn.getHostname(), sn);
245     }
246 
247     for (String host : hosts) {
248       ServerName sn = hostToServerName.get(host);
249       // it is possible that HDFS is up ( thus host is valid ),
250       // but RS is down ( thus sn is null )
251       if (sn != null) {
252         topServerNames.add(sn);
253       }
254     }
255     return topServerNames;
256   }
257 
258   public HDFSBlocksDistribution getBlockDistribution(HRegionInfo hri) {
259     HDFSBlocksDistribution blockDistbn = null;
260     try {
261       if (cache.asMap().containsKey(hri)) {
262         blockDistbn = cache.get(hri);
263         return blockDistbn;
264       } else {
265         LOG.debug("HDFSBlocksDistribution not found in cache for region "
266             + hri.getRegionNameAsString());
267         blockDistbn = internalGetTopBlockLocation(hri);
268         cache.put(hri, blockDistbn);
269         return blockDistbn;
270       }
271     } catch (ExecutionException e) {
272       LOG.warn("Error while fetching cache entry ", e);
273       blockDistbn = internalGetTopBlockLocation(hri);
274       cache.put(hri, blockDistbn);
275       return blockDistbn;
276     }
277   }
278 }