View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.rest;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.IOException;
24  import java.io.StringWriter;
25  import java.util.Iterator;
26  import java.util.Random;
27  
28  import javax.xml.bind.JAXBContext;
29  import javax.xml.bind.JAXBException;
30  import javax.xml.bind.Marshaller;
31  import javax.xml.bind.Unmarshaller;
32  
33  import org.apache.commons.httpclient.Header;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.HBaseAdmin;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Durability;
40  import org.apache.hadoop.hbase.rest.client.Client;
41  import org.apache.hadoop.hbase.rest.client.Cluster;
42  import org.apache.hadoop.hbase.rest.client.Response;
43  import org.apache.hadoop.hbase.rest.model.CellModel;
44  import org.apache.hadoop.hbase.rest.model.CellSetModel;
45  import org.apache.hadoop.hbase.rest.model.RowModel;
46  import org.apache.hadoop.hbase.rest.model.ScannerModel;
47  import org.apache.hadoop.hbase.util.Bytes;
48  
49  import static org.junit.Assert.*;
50  
51  import org.junit.AfterClass;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  @Category(MediumTests.class)
57  public class TestScannerResource {
58    private static final String TABLE = "TestScannerResource";
59    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60    private static final String CFA = "a";
61    private static final String CFB = "b";
62    private static final String COLUMN_1 = CFA + ":1";
63    private static final String COLUMN_2 = CFB + ":2";
64  
65    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
67      new HBaseRESTTestingUtility();
68    private static Client client;
69    private static JAXBContext context;
70    private static Marshaller marshaller;
71    private static Unmarshaller unmarshaller;
72    private static int expectedRows1;
73    private static int expectedRows2;
74    private static Configuration conf;
75  
76    private static int insertData(String tableName, String column, double prob)
77        throws IOException {
78      Random rng = new Random();
79      int count = 0;
80      HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
81      byte[] k = new byte[3];
82      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83      for (byte b1 = 'a'; b1 < 'z'; b1++) {
84        for (byte b2 = 'a'; b2 < 'z'; b2++) {
85          for (byte b3 = 'a'; b3 < 'z'; b3++) {
86            if (rng.nextDouble() < prob) {
87              k[0] = b1;
88              k[1] = b2;
89              k[2] = b3;
90              Put put = new Put(k);
91              put.setDurability(Durability.SKIP_WAL);
92              put.add(famAndQf[0], famAndQf[1], k);
93              table.put(put);
94              count++;
95            }
96          }
97        }
98      }
99      table.flushCommits();
100     return count;
101   }
102 
103   private static int countCellSet(CellSetModel model) {
104     int count = 0;
105     Iterator<RowModel> rows = model.getRows().iterator();
106     while (rows.hasNext()) {
107       RowModel row = rows.next();
108       Iterator<CellModel> cells = row.getCells().iterator();
109       while (cells.hasNext()) {
110         cells.next();
111         count++;
112       }
113     }
114     return count;
115   }
116 
117   private static int fullTableScan(ScannerModel model) throws IOException {
118     model.setBatch(100);
119     Response response = client.put("/" + TABLE + "/scanner",
120       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
121     assertEquals(response.getCode(), 201);
122     String scannerURI = response.getLocation();
123     assertNotNull(scannerURI);
124     int count = 0;
125     while (true) {
126       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
127       assertTrue(response.getCode() == 200 || response.getCode() == 204);
128       if (response.getCode() == 200) {
129         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
130         CellSetModel cellSet = new CellSetModel();
131         cellSet.getObjectFromMessage(response.getBody());
132         Iterator<RowModel> rows = cellSet.getRows().iterator();
133         while (rows.hasNext()) {
134           RowModel row = rows.next();
135           Iterator<CellModel> cells = row.getCells().iterator();
136           while (cells.hasNext()) {
137             cells.next();
138             count++;
139           }
140         }
141       } else {
142         break;
143       }
144     }
145     // delete the scanner
146     response = client.delete(scannerURI);
147     assertEquals(response.getCode(), 200);
148     return count;
149   }
150 
151   @BeforeClass
152   public static void setUpBeforeClass() throws Exception {
153     conf = TEST_UTIL.getConfiguration();
154     TEST_UTIL.startMiniCluster();
155     REST_TEST_UTIL.startServletContainer(conf);
156     client = new Client(new Cluster().add("localhost",
157       REST_TEST_UTIL.getServletPort()));
158     context = JAXBContext.newInstance(
159       CellModel.class,
160       CellSetModel.class,
161       RowModel.class,
162       ScannerModel.class);
163     marshaller = context.createMarshaller();
164     unmarshaller = context.createUnmarshaller();
165     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
166     if (admin.tableExists(TABLE)) {
167       return;
168     }
169     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
170     htd.addFamily(new HColumnDescriptor(CFA));
171     htd.addFamily(new HColumnDescriptor(CFB));
172     admin.createTable(htd);
173     expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
174     expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
175   }
176 
177   @AfterClass
178   public static void tearDownAfterClass() throws Exception {
179     REST_TEST_UTIL.shutdownServletContainer();
180     TEST_UTIL.shutdownMiniCluster();
181   }
182 
183   @Test
184   public void testSimpleScannerXML() throws IOException, JAXBException {
185     final int BATCH_SIZE = 5;
186     // new scanner
187     ScannerModel model = new ScannerModel();
188     model.setBatch(BATCH_SIZE);
189     model.addColumn(Bytes.toBytes(COLUMN_1));
190     StringWriter writer = new StringWriter();
191     marshaller.marshal(model, writer);
192     byte[] body = Bytes.toBytes(writer.toString());
193 
194     // test put operation is forbidden in read-only mode
195     conf.set("hbase.rest.readonly", "true");
196     Response response = client.put("/" + TABLE + "/scanner",
197       Constants.MIMETYPE_XML, body);
198     assertEquals(response.getCode(), 403);
199     String scannerURI = response.getLocation();
200     assertNull(scannerURI);
201 
202     // recall previous put operation with read-only off
203     conf.set("hbase.rest.readonly", "false");
204     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
205       body);
206     assertEquals(response.getCode(), 201);
207     scannerURI = response.getLocation();
208     assertNotNull(scannerURI);
209 
210     // get a cell set
211     response = client.get(scannerURI, Constants.MIMETYPE_XML);
212     assertEquals(response.getCode(), 200);
213     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
214     CellSetModel cellSet = (CellSetModel)
215       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
216     // confirm batch size conformance
217     assertEquals(countCellSet(cellSet), BATCH_SIZE);
218 
219     // test delete scanner operation is forbidden in read-only mode
220     conf.set("hbase.rest.readonly", "true");
221     response = client.delete(scannerURI);
222     assertEquals(response.getCode(), 403);
223 
224     // recall previous delete scanner operation with read-only off
225     conf.set("hbase.rest.readonly", "false");
226     response = client.delete(scannerURI);
227     assertEquals(response.getCode(), 200);
228   }
229 
230   @Test
231   public void testSimpleScannerPB() throws IOException {
232     final int BATCH_SIZE = 10;
233     // new scanner
234     ScannerModel model = new ScannerModel();
235     model.setBatch(BATCH_SIZE);
236     model.addColumn(Bytes.toBytes(COLUMN_1));
237 
238     // test put operation is forbidden in read-only mode
239     conf.set("hbase.rest.readonly", "true");
240     Response response = client.put("/" + TABLE + "/scanner",
241       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
242     assertEquals(response.getCode(), 403);
243     String scannerURI = response.getLocation();
244     assertNull(scannerURI);
245 
246     // recall previous put operation with read-only off
247     conf.set("hbase.rest.readonly", "false");
248     response = client.put("/" + TABLE + "/scanner",
249       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
250     assertEquals(response.getCode(), 201);
251     scannerURI = response.getLocation();
252     assertNotNull(scannerURI);
253 
254     // get a cell set
255     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
256     assertEquals(response.getCode(), 200);
257     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
258     CellSetModel cellSet = new CellSetModel();
259     cellSet.getObjectFromMessage(response.getBody());
260     // confirm batch size conformance
261     assertEquals(countCellSet(cellSet), BATCH_SIZE);
262 
263     // test delete scanner operation is forbidden in read-only mode
264     conf.set("hbase.rest.readonly", "true");
265     response = client.delete(scannerURI);
266     assertEquals(response.getCode(), 403);
267 
268     // recall previous delete scanner operation with read-only off
269     conf.set("hbase.rest.readonly", "false");
270     response = client.delete(scannerURI);
271     assertEquals(response.getCode(), 200);
272   }
273 
274   @Test
275   public void testSimpleScannerBinary() throws IOException {
276     // new scanner
277     ScannerModel model = new ScannerModel();
278     model.setBatch(1);
279     model.addColumn(Bytes.toBytes(COLUMN_1));
280 
281     // test put operation is forbidden in read-only mode
282     conf.set("hbase.rest.readonly", "true");
283     Response response = client.put("/" + TABLE + "/scanner",
284       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
285     assertEquals(response.getCode(), 403);
286     String scannerURI = response.getLocation();
287     assertNull(scannerURI);
288 
289     // recall previous put operation with read-only off
290     conf.set("hbase.rest.readonly", "false");
291     response = client.put("/" + TABLE + "/scanner",
292       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
293     assertEquals(response.getCode(), 201);
294     scannerURI = response.getLocation();
295     assertNotNull(scannerURI);
296 
297     // get a cell
298     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
299     assertEquals(response.getCode(), 200);
300     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
301     // verify that data was returned
302     assertTrue(response.getBody().length > 0);
303     // verify that the expected X-headers are present
304     boolean foundRowHeader = false, foundColumnHeader = false,
305       foundTimestampHeader = false;
306     for (Header header: response.getHeaders()) {
307       if (header.getName().equals("X-Row")) {
308         foundRowHeader = true;
309       } else if (header.getName().equals("X-Column")) {
310         foundColumnHeader = true;
311       } else if (header.getName().equals("X-Timestamp")) {
312         foundTimestampHeader = true;
313       }
314     }
315     assertTrue(foundRowHeader);
316     assertTrue(foundColumnHeader);
317     assertTrue(foundTimestampHeader);
318 
319     // test delete scanner operation is forbidden in read-only mode
320     conf.set("hbase.rest.readonly", "true");
321     response = client.delete(scannerURI);
322     assertEquals(response.getCode(), 403);
323 
324     // recall previous delete scanner operation with read-only off
325     conf.set("hbase.rest.readonly", "false");
326     response = client.delete(scannerURI);
327     assertEquals(response.getCode(), 200);
328   }
329 
330   @Test
331   public void testFullTableScan() throws IOException {
332     ScannerModel model = new ScannerModel();
333     model.addColumn(Bytes.toBytes(COLUMN_1));
334     assertEquals(fullTableScan(model), expectedRows1);
335 
336     model = new ScannerModel();
337     model.addColumn(Bytes.toBytes(COLUMN_2));
338     assertEquals(fullTableScan(model), expectedRows2);
339   }
340 
341   @Test
342   public void testTableDoesNotExist() throws IOException, JAXBException {
343     ScannerModel model = new ScannerModel();
344     StringWriter writer = new StringWriter();
345     marshaller.marshal(model, writer);
346     byte[] body = Bytes.toBytes(writer.toString());
347     Response response = client.put("/" + NONEXISTENT_TABLE +
348       "/scanner", Constants.MIMETYPE_XML, body);
349     assertEquals(response.getCode(), 404);
350   }
351 
352 }
353