1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.List;
25 import java.util.ArrayList;
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.client.HTable;
30 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
31 import org.apache.hadoop.hbase.regionserver.HRegionServer;
32 import org.apache.hadoop.hbase.regionserver.HRegion;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.util.JVMClusterUtil;
36 import org.apache.hadoop.hbase.util.Threads;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39
40
41
42
43
44 @Category(MediumTests.class)
45 public class TestGlobalMemStoreSize {
46 private final Log LOG = LogFactory.getLog(this.getClass().getName());
47 private static int regionServerNum = 4;
48 private static int regionNum = 16;
49
50 private static int totalRegionNum = regionNum+2;
51
52 private HBaseTestingUtility TEST_UTIL;
53 private MiniHBaseCluster cluster;
54
55
56
57
58
59
60 @Test
61 public void testGlobalMemStore() throws Exception {
62
63 LOG.info("Starting cluster");
64 Configuration conf = HBaseConfiguration.create();
65 TEST_UTIL = new HBaseTestingUtility(conf);
66 TEST_UTIL.startMiniCluster(1, regionServerNum);
67 cluster = TEST_UTIL.getHBaseCluster();
68 LOG.info("Waiting for active/ready master");
69 cluster.waitForActiveAndReadyMaster();
70
71
72 byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
73 byte [] family = Bytes.toBytes("family");
74 LOG.info("Creating table with " + regionNum + " regions");
75 HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), family);
76 int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family,
77 regionNum);
78 assertEquals(regionNum,numRegions);
79 waitForAllRegionsAssigned();
80
81 for (HRegionServer server : getOnlineRegionServers()) {
82 long globalMemStoreSize = 0;
83 for (HRegionInfo regionInfo :
84 ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
85 globalMemStoreSize +=
86 server.getFromOnlineRegions(regionInfo.getEncodedName()).
87 getMemstoreSize().get();
88 }
89 assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
90 globalMemStoreSize);
91 }
92
93
94 int i = 0;
95 for (HRegionServer server : getOnlineRegionServers()) {
96 LOG.info("Starting flushes on " + server.getServerName() +
97 ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
98
99 for (HRegionInfo regionInfo :
100 ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
101 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
102 flush(r, server);
103 }
104 LOG.info("Post flush on " + server.getServerName());
105 long now = System.currentTimeMillis();
106 long timeout = now + 1000;
107 while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
108 timeout < System.currentTimeMillis()) {
109 Threads.sleep(10);
110 }
111 long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
112 if (size > 0) {
113
114
115 for (HRegionInfo regionInfo :
116 ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
117 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
118 long l = r.getMemstoreSize().longValue();
119 if (l > 0) {
120
121
122 assertTrue(regionInfo.isMetaRegion());
123 LOG.info(r.toString() + " " + l + ", reflushing");
124 r.flushcache();
125 }
126 }
127 }
128 size = server.getRegionServerAccounting().getGlobalMemstoreSize();
129 assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
130 }
131
132 ht.close();
133 TEST_UTIL.shutdownMiniCluster();
134 }
135
136
137
138
139
140
141
142 private void flush(final HRegion r, final HRegionServer server)
143 throws IOException {
144 LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
145 ", " + r.flushcache() + ", size=" +
146 server.getRegionServerAccounting().getGlobalMemstoreSize());
147 }
148
149 private List<HRegionServer> getOnlineRegionServers() {
150 List<HRegionServer> list = new ArrayList<HRegionServer>();
151 for (JVMClusterUtil.RegionServerThread rst :
152 cluster.getRegionServerThreads()) {
153 if (rst.getRegionServer().isOnline()) {
154 list.add(rst.getRegionServer());
155 }
156 }
157 return list;
158 }
159
160
161
162
163 private void waitForAllRegionsAssigned() throws IOException {
164 while (true) {
165 int regionCount = HBaseTestingUtility.getAllOnlineRegions(cluster).size();
166 if (regionCount >= totalRegionNum) break;
167 LOG.debug("Waiting for there to be "+ totalRegionNum
168 +" regions, but there are " + regionCount + " right now.");
169 try {
170 Thread.sleep(100);
171 } catch (InterruptedException e) {}
172 }
173 }
174 }