1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.rest;
22  
23  import java.io.ByteArrayInputStream;
24  import java.io.IOException;
25  import java.io.StringWriter;
26  import java.util.Iterator;
27  import java.util.Random;
28  
29  import javax.xml.bind.JAXBContext;
30  import javax.xml.bind.JAXBException;
31  import javax.xml.bind.Marshaller;
32  import javax.xml.bind.Unmarshaller;
33  
34  import org.apache.commons.httpclient.Header;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.*;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.rest.client.Client;
41  import org.apache.hadoop.hbase.rest.client.Cluster;
42  import org.apache.hadoop.hbase.rest.client.Response;
43  import org.apache.hadoop.hbase.rest.model.CellModel;
44  import org.apache.hadoop.hbase.rest.model.CellSetModel;
45  import org.apache.hadoop.hbase.rest.model.RowModel;
46  import org.apache.hadoop.hbase.rest.model.ScannerModel;
47  import org.apache.hadoop.hbase.util.Bytes;
48  
49  import static org.junit.Assert.*;
50  import org.junit.AfterClass;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  @Category(MediumTests.class)
56  public class TestScannerResource {
57    private static final String TABLE = "TestScannerResource";
58    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
59    private static final String CFA = "a";
60    private static final String CFB = "b";
61    private static final String COLUMN_1 = CFA + ":1";
62    private static final String COLUMN_2 = CFB + ":2";
63  
64    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
66      new HBaseRESTTestingUtility();
67    private static Client client;
68    private static JAXBContext context;
69    private static Marshaller marshaller;
70    private static Unmarshaller unmarshaller;
71    private static int expectedRows1;
72    private static int expectedRows2;
73    private static Configuration conf;
74  
75    private static int insertData(String tableName, String column, double prob)
76        throws IOException {
77      Random rng = new Random();
78      int count = 0;
79      HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
80      byte[] k = new byte[3];
81      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
82      for (byte b1 = 'a'; b1 < 'z'; b1++) {
83        for (byte b2 = 'a'; b2 < 'z'; b2++) {
84          for (byte b3 = 'a'; b3 < 'z'; b3++) {
85            if (rng.nextDouble() < prob) {
86              k[0] = b1;
87              k[1] = b2;
88              k[2] = b3;
89              Put put = new Put(k);
90              put.setWriteToWAL(false);
91              put.add(famAndQf[0], famAndQf[1], k);
92              table.put(put);
93              count++;
94            }
95          }
96        }
97      }
98      table.flushCommits();
99      return count;
100   }
101 
102   private static int countCellSet(CellSetModel model) {
103     int count = 0;
104     Iterator<RowModel> rows = model.getRows().iterator();
105     while (rows.hasNext()) {
106       RowModel row = rows.next();
107       Iterator<CellModel> cells = row.getCells().iterator();
108       while (cells.hasNext()) {
109         cells.next();
110         count++;
111       }
112     }
113     return count;
114   }
115 
116   private static int fullTableScan(ScannerModel model) throws IOException {
117     model.setBatch(100);
118     Response response = client.put("/" + TABLE + "/scanner",
119       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
120     assertEquals(response.getCode(), 201);
121     String scannerURI = response.getLocation();
122     assertNotNull(scannerURI);
123     int count = 0;
124     while (true) {
125       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
126       assertTrue(response.getCode() == 200 || response.getCode() == 204);
127       if (response.getCode() == 200) {
128         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
129         CellSetModel cellSet = new CellSetModel();
130         cellSet.getObjectFromMessage(response.getBody());
131         Iterator<RowModel> rows = cellSet.getRows().iterator();
132         while (rows.hasNext()) {
133           RowModel row = rows.next();
134           Iterator<CellModel> cells = row.getCells().iterator();
135           while (cells.hasNext()) {
136             cells.next();
137             count++;
138           }
139         }
140       } else {
141         break;
142       }
143     }
144     // delete the scanner
145     response = client.delete(scannerURI);
146     assertEquals(response.getCode(), 200);
147     return count;
148   }
149 
150   @BeforeClass
151   public static void setUpBeforeClass() throws Exception {
152     conf = TEST_UTIL.getConfiguration();
153     TEST_UTIL.startMiniCluster();
154     REST_TEST_UTIL.startServletContainer(conf);
155     client = new Client(new Cluster().add("localhost",
156       REST_TEST_UTIL.getServletPort()));
157     context = JAXBContext.newInstance(
158       CellModel.class,
159       CellSetModel.class,
160       RowModel.class,
161       ScannerModel.class);
162     marshaller = context.createMarshaller();
163     unmarshaller = context.createUnmarshaller();
164     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
165     if (admin.tableExists(TABLE)) {
166       return;
167     }
168     HTableDescriptor htd = new HTableDescriptor(TABLE);
169     htd.addFamily(new HColumnDescriptor(CFA));
170     htd.addFamily(new HColumnDescriptor(CFB));
171     admin.createTable(htd);
172     expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
173     expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
174   }
175 
176   @AfterClass
177   public static void tearDownAfterClass() throws Exception {
178     REST_TEST_UTIL.shutdownServletContainer();
179     TEST_UTIL.shutdownMiniCluster();
180   }
181 
182   @Test
183   public void testSimpleScannerXML() throws IOException, JAXBException {
184     final int BATCH_SIZE = 5;
185     // new scanner
186     ScannerModel model = new ScannerModel();
187     model.setBatch(BATCH_SIZE);
188     model.addColumn(Bytes.toBytes(COLUMN_1));
189     StringWriter writer = new StringWriter();
190     marshaller.marshal(model, writer);
191     byte[] body = Bytes.toBytes(writer.toString());
192 
193     // test put operation is forbidden in read-only mode
194     conf.set("hbase.rest.readonly", "true");
195     Response response = client.put("/" + TABLE + "/scanner",
196       Constants.MIMETYPE_XML, body);
197     assertEquals(response.getCode(), 403);
198     String scannerURI = response.getLocation();
199     assertNull(scannerURI);
200 
201     // recall previous put operation with read-only off
202     conf.set("hbase.rest.readonly", "false");
203     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
204       body);
205     assertEquals(response.getCode(), 201);
206     scannerURI = response.getLocation();
207     assertNotNull(scannerURI);
208 
209     // get a cell set
210     response = client.get(scannerURI, Constants.MIMETYPE_XML);
211     assertEquals(response.getCode(), 200);
212     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
213     CellSetModel cellSet = (CellSetModel)
214       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
215     // confirm batch size conformance
216     assertEquals(countCellSet(cellSet), BATCH_SIZE);
217 
218     // test delete scanner operation is forbidden in read-only mode
219     conf.set("hbase.rest.readonly", "true");
220     response = client.delete(scannerURI);
221     assertEquals(response.getCode(), 403);
222 
223     // recall previous delete scanner operation with read-only off
224     conf.set("hbase.rest.readonly", "false");
225     response = client.delete(scannerURI);
226     assertEquals(response.getCode(), 200);
227   }
228 
229   @Test
230   public void testSimpleScannerPB() throws IOException {
231     final int BATCH_SIZE = 10;
232     // new scanner
233     ScannerModel model = new ScannerModel();
234     model.setBatch(BATCH_SIZE);
235     model.addColumn(Bytes.toBytes(COLUMN_1));
236 
237     // test put operation is forbidden in read-only mode
238     conf.set("hbase.rest.readonly", "true");
239     Response response = client.put("/" + TABLE + "/scanner",
240       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
241     assertEquals(response.getCode(), 403);
242     String scannerURI = response.getLocation();
243     assertNull(scannerURI);
244 
245     // recall previous put operation with read-only off
246     conf.set("hbase.rest.readonly", "false");
247     response = client.put("/" + TABLE + "/scanner",
248       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
249     assertEquals(response.getCode(), 201);
250     scannerURI = response.getLocation();
251     assertNotNull(scannerURI);
252 
253     // get a cell set
254     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
255     assertEquals(response.getCode(), 200);
256     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
257     CellSetModel cellSet = new CellSetModel();
258     cellSet.getObjectFromMessage(response.getBody());
259     // confirm batch size conformance
260     assertEquals(countCellSet(cellSet), BATCH_SIZE);
261 
262     // test delete scanner operation is forbidden in read-only mode
263     conf.set("hbase.rest.readonly", "true");
264     response = client.delete(scannerURI);
265     assertEquals(response.getCode(), 403);
266 
267     // recall previous delete scanner operation with read-only off
268     conf.set("hbase.rest.readonly", "false");
269     response = client.delete(scannerURI);
270     assertEquals(response.getCode(), 200);
271   }
272 
273   @Test
274   public void testSimpleScannerBinary() throws IOException {
275     // new scanner
276     ScannerModel model = new ScannerModel();
277     model.setBatch(1);
278     model.addColumn(Bytes.toBytes(COLUMN_1));
279 
280     // test put operation is forbidden in read-only mode
281     conf.set("hbase.rest.readonly", "true");
282     Response response = client.put("/" + TABLE + "/scanner",
283       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
284     assertEquals(response.getCode(), 403);
285     String scannerURI = response.getLocation();
286     assertNull(scannerURI);
287 
288     // recall previous put operation with read-only off
289     conf.set("hbase.rest.readonly", "false");
290     response = client.put("/" + TABLE + "/scanner",
291       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
292     assertEquals(response.getCode(), 201);
293     scannerURI = response.getLocation();
294     assertNotNull(scannerURI);
295 
296     // get a cell
297     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
298     assertEquals(response.getCode(), 200);
299     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
300     // verify that data was returned
301     assertTrue(response.getBody().length > 0);
302     // verify that the expected X-headers are present
303     boolean foundRowHeader = false, foundColumnHeader = false,
304       foundTimestampHeader = false;
305     for (Header header: response.getHeaders()) {
306       if (header.getName().equals("X-Row")) {
307         foundRowHeader = true;
308       } else if (header.getName().equals("X-Column")) {
309         foundColumnHeader = true;
310       } else if (header.getName().equals("X-Timestamp")) {
311         foundTimestampHeader = true;
312       }
313     }
314     assertTrue(foundRowHeader);
315     assertTrue(foundColumnHeader);
316     assertTrue(foundTimestampHeader);
317 
318     // test delete scanner operation is forbidden in read-only mode
319     conf.set("hbase.rest.readonly", "true");
320     response = client.delete(scannerURI);
321     assertEquals(response.getCode(), 403);
322 
323     // recall previous delete scanner operation with read-only off
324     conf.set("hbase.rest.readonly", "false");
325     response = client.delete(scannerURI);
326     assertEquals(response.getCode(), 200);
327   }
328 
329   @Test
330   public void testFullTableScan() throws IOException {
331     ScannerModel model = new ScannerModel();
332     model.addColumn(Bytes.toBytes(COLUMN_1));
333     assertEquals(fullTableScan(model), expectedRows1);
334 
335     model = new ScannerModel();
336     model.addColumn(Bytes.toBytes(COLUMN_2));
337     assertEquals(fullTableScan(model), expectedRows2);
338   }
339 
340   @Test
341   public void testTableDoesNotExist() throws IOException, JAXBException {
342     ScannerModel model = new ScannerModel();
343     StringWriter writer = new StringWriter();
344     marshaller.marshal(model, writer);
345     byte[] body = Bytes.toBytes(writer.toString());
346     Response response = client.put("/" + NONEXISTENT_TABLE +
347       "/scanner", Constants.MIMETYPE_XML, body);
348     assertEquals(response.getCode(), 404);
349   }
350 
351   @org.junit.Rule
352   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
353     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
354 }
355