1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.zip.Checksum;
24
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.util.ByteBufferUtils;
28 import org.apache.hadoop.hbase.util.Bytes;
29 import org.apache.hadoop.hbase.util.ChecksumType;
30
31
32
33
34 @InterfaceAudience.Private
35 public class ChecksumUtil {
36
37
38 private static byte[] DUMMY_VALUE = new byte[128 * HFileBlock.CHECKSUM_SIZE];
39
40
41
42
43
44
45
46
47 private static boolean generateExceptions = false;
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63 static void generateChecksums(byte[] indata,
64 int startOffset, int endOffset,
65 byte[] outdata, int outOffset,
66 ChecksumType checksumType,
67 int bytesPerChecksum) throws IOException {
68
69 if (checksumType == ChecksumType.NULL) {
70 return;
71 }
72
73 Checksum checksum = checksumType.getChecksumObject();
74 int bytesLeft = endOffset - startOffset;
75 int chunkNum = 0;
76
77 while (bytesLeft > 0) {
78
79 checksum.reset();
80 int count = Math.min(bytesLeft, bytesPerChecksum);
81 checksum.update(indata, startOffset, count);
82
83
84 int cksumValue = (int)checksum.getValue();
85 outOffset = Bytes.putInt(outdata, outOffset, cksumValue);
86 chunkNum++;
87 startOffset += count;
88 bytesLeft -= count;
89 }
90 }
91
92
93
94
95
96
97
98
99
100
101 static boolean validateBlockChecksum(Path path, HFileBlock block,
102 byte[] data, int hdrSize) throws IOException {
103
104
105
106
107
108
109
110 if (!block.getHFileContext().isUseHBaseChecksum()) {
111 return false;
112 }
113
114
115
116
117
118 ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
119 if (cktype == ChecksumType.NULL) {
120 return true;
121 }
122 Checksum checksumObject = cktype.getChecksumObject();
123 checksumObject.reset();
124
125
126 int bytesPerChecksum = block.getBytesPerChecksum();
127
128
129 if (bytesPerChecksum < hdrSize) {
130 String msg = "Unsupported value of bytesPerChecksum. " +
131 " Minimum is " + hdrSize +
132 " but the configured value is " + bytesPerChecksum;
133 HFile.LOG.warn(msg);
134 return false;
135 }
136
137 ByteBuffer hdr = block.getBufferWithHeader();
138 if (hdr.hasArray()) {
139 checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize);
140 } else {
141 checksumObject.update(ByteBufferUtils.toBytes(hdr, 0, hdrSize), 0, hdrSize);
142 }
143
144 int off = hdrSize;
145 int consumed = hdrSize;
146 int bytesLeft = block.getOnDiskDataSizeWithHeader() - off;
147 int cksumOffset = block.getOnDiskDataSizeWithHeader();
148
149
150 while (bytesLeft > 0) {
151 int thisChunkSize = bytesPerChecksum - consumed;
152 int count = Math.min(bytesLeft, thisChunkSize);
153 checksumObject.update(data, off, count);
154
155 int storedChecksum = Bytes.toInt(data, cksumOffset);
156 if (storedChecksum != (int)checksumObject.getValue()) {
157 String msg = "File " + path +
158 " Stored checksum value of " + storedChecksum +
159 " at offset " + cksumOffset +
160 " does not match computed checksum " +
161 checksumObject.getValue() +
162 ", total data size " + data.length +
163 " Checksum data range offset " + off + " len " + count +
164 HFileBlock.toStringHeader(block.getBufferReadOnly());
165 HFile.LOG.warn(msg);
166 if (generateExceptions) {
167 throw new IOException(msg);
168 } else {
169 return false;
170 }
171 }
172 cksumOffset += HFileBlock.CHECKSUM_SIZE;
173 bytesLeft -= count;
174 off += count;
175 consumed = 0;
176 checksumObject.reset();
177 }
178 return true;
179 }
180
181
182
183
184
185
186
187
188 static long numBytes(long datasize, int bytesPerChecksum) {
189 return numChunks(datasize, bytesPerChecksum) *
190 HFileBlock.CHECKSUM_SIZE;
191 }
192
193
194
195
196
197
198
199
200 static long numChunks(long datasize, int bytesPerChecksum) {
201 long numChunks = datasize/bytesPerChecksum;
202 if (datasize % bytesPerChecksum != 0) {
203 numChunks++;
204 }
205 return numChunks;
206 }
207
208
209
210
211
212
213
214
215
216 static void reserveSpaceForChecksums(ByteArrayOutputStream baos,
217 int numBytes, int bytesPerChecksum) throws IOException {
218 long numChunks = numChunks(numBytes, bytesPerChecksum);
219 long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE;
220 while (bytesLeft > 0) {
221 long count = Math.min(bytesLeft, DUMMY_VALUE.length);
222 baos.write(DUMMY_VALUE, 0, (int)count);
223 bytesLeft -= count;
224 }
225 }
226
227
228
229
230
231
232
233 public static void generateExceptionForChecksumFailureForTest(boolean value) {
234 generateExceptions = value;
235 }
236 }
237