1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20 import org.apache.hadoop.classification.InterfaceAudience;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25
26 import org.apache.hadoop.hbase.HBaseConfiguration;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
29 import org.apache.hadoop.hbase.regionserver.wal.HLog;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.io.WritableUtils;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import com.google.common.base.Preconditions;
36 import com.google.protobuf.ByteString;
37
38
39
40
41
42 @InterfaceAudience.Private
43 public class Compressor {
44
45
46
47 public static void main(String[] args) throws IOException {
48 if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) {
49 printHelp();
50 System.exit(-1);
51 }
52
53 Path inputPath = new Path(args[0]);
54 Path outputPath = new Path(args[1]);
55
56 transformFile(inputPath, outputPath);
57 }
58
59 private static void printHelp() {
60 System.err.println("usage: Compressor <input> <output>");
61 System.err.println("If <input> HLog is compressed, <output> will be decompressed.");
62 System.err.println("If <input> HLog is uncompressed, <output> will be compressed.");
63 return;
64 }
65
66 private static void transformFile(Path input, Path output)
67 throws IOException {
68 Configuration conf = HBaseConfiguration.create();
69
70 FileSystem inFS = input.getFileSystem(conf);
71 FileSystem outFS = output.getFileSystem(conf);
72
73 HLog.Reader in = HLogFactory.createReader(inFS, input, conf, null, false);
74 HLog.Writer out = null;
75
76 try {
77 if (!(in instanceof ReaderBase)) {
78 System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
79 return;
80 }
81 boolean compress = ((ReaderBase)in).hasCompression();
82 conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
83 out = HLogFactory.createWriter(outFS, output, conf);
84
85 HLog.Entry e = null;
86 while ((e = in.next()) != null) out.append(e);
87 } finally {
88 in.close();
89 if (out != null) {
90 out.close();
91 out = null;
92 }
93 }
94 }
95
96
97
98
99
100
101
102
103 @Deprecated
104 static byte[] readCompressed(DataInput in, Dictionary dict)
105 throws IOException {
106 byte status = in.readByte();
107
108 if (status == Dictionary.NOT_IN_DICTIONARY) {
109 int length = WritableUtils.readVInt(in);
110
111 byte[] arr = new byte[length];
112 in.readFully(arr);
113 if (dict != null) dict.addEntry(arr, 0, length);
114 return arr;
115 } else {
116
117
118
119 short dictIdx = toShort(status, in.readByte());
120 byte[] entry = dict.getEntry(dictIdx);
121 if (entry == null) {
122 throw new IOException("Missing dictionary entry for index "
123 + dictIdx);
124 }
125 return entry;
126 }
127 }
128
129
130
131
132
133
134
135
136
137
138
139
140 @Deprecated
141 static int uncompressIntoArray(byte[] to, int offset, DataInput in,
142 Dictionary dict) throws IOException {
143 byte status = in.readByte();
144
145 if (status == Dictionary.NOT_IN_DICTIONARY) {
146
147
148 int length = WritableUtils.readVInt(in);
149 in.readFully(to, offset, length);
150 dict.addEntry(to, offset, length);
151 return length;
152 } else {
153
154
155 short dictIdx = toShort(status, in.readByte());
156 byte[] entry;
157 try {
158 entry = dict.getEntry(dictIdx);
159 } catch (Exception ex) {
160 throw new IOException("Unable to uncompress the log entry", ex);
161 }
162 if (entry == null) {
163 throw new IOException("Missing dictionary entry for index "
164 + dictIdx);
165 }
166
167 Bytes.putBytes(to, offset, entry, 0, entry.length);
168 return entry.length;
169 }
170 }
171
172
173
174
175
176
177
178
179 @Deprecated
180 static void writeCompressed(byte[] data, int offset, int length,
181 DataOutput out, Dictionary dict)
182 throws IOException {
183 short dictIdx = Dictionary.NOT_IN_DICTIONARY;
184 if (dict != null) {
185 dictIdx = dict.findEntry(data, offset, length);
186 }
187 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
188
189 out.writeByte(Dictionary.NOT_IN_DICTIONARY);
190 WritableUtils.writeVInt(out, length);
191 out.write(data, offset, length);
192 } else {
193 out.writeShort(dictIdx);
194 }
195 }
196
197 static short toShort(byte hi, byte lo) {
198 short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
199 Preconditions.checkArgument(s >= 0);
200 return s;
201 }
202 }