1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20 import java.io.DataInput;
21 import java.io.DataOutput;
22 import java.io.IOException;
23
24 import org.apache.hadoop.classification.InterfaceAudience;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.io.WritableUtils;
32
33 import com.google.common.base.Preconditions;
34
35
36
37
38
39 @InterfaceAudience.Private
40 public class Compressor {
41
42
43
44 public static void main(String[] args) throws IOException {
45 if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) {
46 printHelp();
47 System.exit(-1);
48 }
49
50 Path inputPath = new Path(args[0]);
51 Path outputPath = new Path(args[1]);
52
53 transformFile(inputPath, outputPath);
54 }
55
56 private static void printHelp() {
57 System.err.println("usage: Compressor <input> <output>");
58 System.err.println("If <input> HLog is compressed, <output> will be decompressed.");
59 System.err.println("If <input> HLog is uncompressed, <output> will be compressed.");
60 return;
61 }
62
63 private static void transformFile(Path input, Path output)
64 throws IOException {
65 Configuration conf = HBaseConfiguration.create();
66
67 FileSystem inFS = input.getFileSystem(conf);
68 FileSystem outFS = output.getFileSystem(conf);
69
70 HLog.Reader in = HLogFactory.createReader(inFS, input, conf, null, false);
71 HLog.Writer out = null;
72
73 try {
74 if (!(in instanceof ReaderBase)) {
75 System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
76 return;
77 }
78 boolean compress = ((ReaderBase)in).hasCompression();
79 conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
80 out = HLogFactory.createWALWriter(outFS, output, conf);
81
82 HLog.Entry e = null;
83 while ((e = in.next()) != null) out.append(e);
84 } finally {
85 in.close();
86 if (out != null) {
87 out.close();
88 out = null;
89 }
90 }
91 }
92
93
94
95
96
97
98
99
100 @Deprecated
101 static byte[] readCompressed(DataInput in, Dictionary dict)
102 throws IOException {
103 byte status = in.readByte();
104
105 if (status == Dictionary.NOT_IN_DICTIONARY) {
106 int length = WritableUtils.readVInt(in);
107
108 byte[] arr = new byte[length];
109 in.readFully(arr);
110 if (dict != null) dict.addEntry(arr, 0, length);
111 return arr;
112 } else {
113
114
115
116 short dictIdx = toShort(status, in.readByte());
117 byte[] entry = dict.getEntry(dictIdx);
118 if (entry == null) {
119 throw new IOException("Missing dictionary entry for index "
120 + dictIdx);
121 }
122 return entry;
123 }
124 }
125
126
127
128
129
130
131
132
133
134
135
136
137 @Deprecated
138 static int uncompressIntoArray(byte[] to, int offset, DataInput in,
139 Dictionary dict) throws IOException {
140 byte status = in.readByte();
141
142 if (status == Dictionary.NOT_IN_DICTIONARY) {
143
144
145 int length = WritableUtils.readVInt(in);
146 in.readFully(to, offset, length);
147 dict.addEntry(to, offset, length);
148 return length;
149 } else {
150
151
152 short dictIdx = toShort(status, in.readByte());
153 byte[] entry;
154 try {
155 entry = dict.getEntry(dictIdx);
156 } catch (Exception ex) {
157 throw new IOException("Unable to uncompress the log entry", ex);
158 }
159 if (entry == null) {
160 throw new IOException("Missing dictionary entry for index "
161 + dictIdx);
162 }
163
164 Bytes.putBytes(to, offset, entry, 0, entry.length);
165 return entry.length;
166 }
167 }
168
169
170
171
172
173
174
175
176 @Deprecated
177 static void writeCompressed(byte[] data, int offset, int length,
178 DataOutput out, Dictionary dict)
179 throws IOException {
180 short dictIdx = Dictionary.NOT_IN_DICTIONARY;
181 if (dict != null) {
182 dictIdx = dict.findEntry(data, offset, length);
183 }
184 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
185
186 out.writeByte(Dictionary.NOT_IN_DICTIONARY);
187 WritableUtils.writeVInt(out, length);
188 out.write(data, offset, length);
189 } else {
190 out.writeShort(dictIdx);
191 }
192 }
193
194 static short toShort(byte hi, byte lo) {
195 short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
196 Preconditions.checkArgument(s >= 0);
197 return s;
198 }
199 }