001 /* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019 package org.apache.commons.compress.compressors.bzip2; 020 021 import java.io.IOException; 022 import java.io.OutputStream; 023 024 import org.apache.commons.compress.compressors.CompressorOutputStream; 025 026 /** 027 * An output stream that compresses into the BZip2 format into another stream. 028 * 029 * <p> 030 * The compression requires large amounts of memory. Thus you should call the 031 * {@link #close() close()} method as soon as possible, to force 032 * <tt>BZip2CompressorOutputStream</tt> to release the allocated memory. 033 * </p> 034 * 035 * <p> You can shrink the amount of allocated memory and maybe raise 036 * the compression speed by choosing a lower blocksize, which in turn 037 * may cause a lower compression ratio. You can avoid unnecessary 038 * memory allocation by avoiding using a blocksize which is bigger 039 * than the size of the input. </p> 040 * 041 * <p> You can compute the memory usage for compressing by the 042 * following formula: </p> 043 * 044 * <pre> 045 * <code>400k + (9 * blocksize)</code>. 046 * </pre> 047 * 048 * <p> To get the memory required for decompression by {@link 049 * BZip2CompressorInputStream} use </p> 050 * 051 * <pre> 052 * <code>65k + (5 * blocksize)</code>. 053 * </pre> 054 * 055 * <table width="100%" border="1"> 056 * <colgroup> <col width="33%" /> <col width="33%" /> <col width="33%" /> 057 * </colgroup> 058 * <tr> 059 * <th colspan="3">Memory usage by blocksize</th> 060 * </tr> 061 * <tr> 062 * <th align="right">Blocksize</th> <th align="right">Compression<br> 063 * memory usage</th> <th align="right">Decompression<br> 064 * memory usage</th> 065 * </tr> 066 * <tr> 067 * <td align="right">100k</td> 068 * <td align="right">1300k</td> 069 * <td align="right">565k</td> 070 * </tr> 071 * <tr> 072 * <td align="right">200k</td> 073 * <td align="right">2200k</td> 074 * <td align="right">1065k</td> 075 * </tr> 076 * <tr> 077 * <td align="right">300k</td> 078 * <td align="right">3100k</td> 079 * <td align="right">1565k</td> 080 * </tr> 081 * <tr> 082 * <td align="right">400k</td> 083 * <td align="right">4000k</td> 084 * <td align="right">2065k</td> 085 * </tr> 086 * <tr> 087 * <td align="right">500k</td> 088 * <td align="right">4900k</td> 089 * <td align="right">2565k</td> 090 * </tr> 091 * <tr> 092 * <td align="right">600k</td> 093 * <td align="right">5800k</td> 094 * <td align="right">3065k</td> 095 * </tr> 096 * <tr> 097 * <td align="right">700k</td> 098 * <td align="right">6700k</td> 099 * <td align="right">3565k</td> 100 * </tr> 101 * <tr> 102 * <td align="right">800k</td> 103 * <td align="right">7600k</td> 104 * <td align="right">4065k</td> 105 * </tr> 106 * <tr> 107 * <td align="right">900k</td> 108 * <td align="right">8500k</td> 109 * <td align="right">4565k</td> 110 * </tr> 111 * </table> 112 * 113 * <p> 114 * For decompression <tt>BZip2CompressorInputStream</tt> allocates less memory if the 115 * bzipped input is smaller than one block. 116 * </p> 117 * 118 * <p> 119 * Instances of this class are not threadsafe. 120 * </p> 121 * 122 * <p> 123 * TODO: Update to BZip2 1.0.1 124 * </p> 125 * @NotThreadSafe 126 */ 127 public class BZip2CompressorOutputStream extends CompressorOutputStream 128 implements BZip2Constants { 129 130 /** 131 * The minimum supported blocksize <tt> == 1</tt>. 132 */ 133 public static final int MIN_BLOCKSIZE = 1; 134 135 /** 136 * The maximum supported blocksize <tt> == 9</tt>. 137 */ 138 public static final int MAX_BLOCKSIZE = 9; 139 140 private static final int SETMASK = (1 << 21); 141 private static final int CLEARMASK = (~SETMASK); 142 private static final int GREATER_ICOST = 15; 143 private static final int LESSER_ICOST = 0; 144 private static final int SMALL_THRESH = 20; 145 private static final int DEPTH_THRESH = 10; 146 private static final int WORK_FACTOR = 30; 147 148 /* 149 * <p> If you are ever unlucky/improbable enough to get a stack 150 * overflow whilst sorting, increase the following constant and 151 * try again. In practice I have never seen the stack go above 27 152 * elems, so the following limit seems very generous. </p> 153 */ 154 private static final int QSORT_STACK_SIZE = 1000; 155 156 /** 157 * Knuth's increments seem to work better than Incerpi-Sedgewick here. 158 * Possibly because the number of elems to sort is usually small, typically 159 * <= 20. 160 */ 161 private static final int[] INCS = { 1, 4, 13, 40, 121, 364, 1093, 3280, 162 9841, 29524, 88573, 265720, 797161, 163 2391484 }; 164 165 private static void hbMakeCodeLengths(final byte[] len, final int[] freq, 166 final Data dat, final int alphaSize, 167 final int maxLen) { 168 /* 169 * Nodes and heap entries run from 1. Entry 0 for both the heap and 170 * nodes is a sentinel. 171 */ 172 final int[] heap = dat.heap; 173 final int[] weight = dat.weight; 174 final int[] parent = dat.parent; 175 176 for (int i = alphaSize; --i >= 0;) { 177 weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; 178 } 179 180 for (boolean tooLong = true; tooLong;) { 181 tooLong = false; 182 183 int nNodes = alphaSize; 184 int nHeap = 0; 185 heap[0] = 0; 186 weight[0] = 0; 187 parent[0] = -2; 188 189 for (int i = 1; i <= alphaSize; i++) { 190 parent[i] = -1; 191 nHeap++; 192 heap[nHeap] = i; 193 194 int zz = nHeap; 195 int tmp = heap[zz]; 196 while (weight[tmp] < weight[heap[zz >> 1]]) { 197 heap[zz] = heap[zz >> 1]; 198 zz >>= 1; 199 } 200 heap[zz] = tmp; 201 } 202 203 while (nHeap > 1) { 204 int n1 = heap[1]; 205 heap[1] = heap[nHeap]; 206 nHeap--; 207 208 int yy = 0; 209 int zz = 1; 210 int tmp = heap[1]; 211 212 while (true) { 213 yy = zz << 1; 214 215 if (yy > nHeap) { 216 break; 217 } 218 219 if ((yy < nHeap) 220 && (weight[heap[yy + 1]] < weight[heap[yy]])) { 221 yy++; 222 } 223 224 if (weight[tmp] < weight[heap[yy]]) { 225 break; 226 } 227 228 heap[zz] = heap[yy]; 229 zz = yy; 230 } 231 232 heap[zz] = tmp; 233 234 int n2 = heap[1]; 235 heap[1] = heap[nHeap]; 236 nHeap--; 237 238 yy = 0; 239 zz = 1; 240 tmp = heap[1]; 241 242 while (true) { 243 yy = zz << 1; 244 245 if (yy > nHeap) { 246 break; 247 } 248 249 if ((yy < nHeap) 250 && (weight[heap[yy + 1]] < weight[heap[yy]])) { 251 yy++; 252 } 253 254 if (weight[tmp] < weight[heap[yy]]) { 255 break; 256 } 257 258 heap[zz] = heap[yy]; 259 zz = yy; 260 } 261 262 heap[zz] = tmp; 263 nNodes++; 264 parent[n1] = parent[n2] = nNodes; 265 266 final int weight_n1 = weight[n1]; 267 final int weight_n2 = weight[n2]; 268 weight[nNodes] = ((weight_n1 & 0xffffff00) 269 + (weight_n2 & 0xffffff00)) 270 | (1 + (((weight_n1 & 0x000000ff) 271 > (weight_n2 & 0x000000ff)) 272 ? (weight_n1 & 0x000000ff) 273 : (weight_n2 & 0x000000ff))); 274 275 parent[nNodes] = -1; 276 nHeap++; 277 heap[nHeap] = nNodes; 278 279 tmp = 0; 280 zz = nHeap; 281 tmp = heap[zz]; 282 final int weight_tmp = weight[tmp]; 283 while (weight_tmp < weight[heap[zz >> 1]]) { 284 heap[zz] = heap[zz >> 1]; 285 zz >>= 1; 286 } 287 heap[zz] = tmp; 288 289 } 290 291 for (int i = 1; i <= alphaSize; i++) { 292 int j = 0; 293 int k = i; 294 295 for (int parent_k; (parent_k = parent[k]) >= 0;) { 296 k = parent_k; 297 j++; 298 } 299 300 len[i - 1] = (byte) j; 301 if (j > maxLen) { 302 tooLong = true; 303 } 304 } 305 306 if (tooLong) { 307 for (int i = 1; i < alphaSize; i++) { 308 int j = weight[i] >> 8; 309 j = 1 + (j >> 1); 310 weight[i] = j << 8; 311 } 312 } 313 } 314 } 315 316 /** 317 * Index of the last char in the block, so the block size == last + 1. 318 */ 319 private int last; 320 321 /** 322 * Index in fmap[] of original string after sorting. 323 */ 324 private int origPtr; 325 326 /** 327 * Always: in the range 0 .. 9. The current block size is 100000 * this 328 * number. 329 */ 330 private final int blockSize100k; 331 332 private boolean blockRandomised; 333 334 private int bsBuff; 335 private int bsLive; 336 private final CRC crc = new CRC(); 337 338 private int nInUse; 339 340 private int nMTF; 341 342 /* 343 * Used when sorting. If too many long comparisons happen, we stop sorting, 344 * randomise the block slightly, and try again. 345 */ 346 private int workDone; 347 private int workLimit; 348 private boolean firstAttempt; 349 350 private int currentChar = -1; 351 private int runLength = 0; 352 353 private int blockCRC; 354 private int combinedCRC; 355 private int allowableBlockSize; 356 357 /** 358 * All memory intensive stuff. 359 */ 360 private Data data; 361 362 private OutputStream out; 363 364 /** 365 * Chooses a blocksize based on the given length of the data to compress. 366 * 367 * @return The blocksize, between {@link #MIN_BLOCKSIZE} and 368 * {@link #MAX_BLOCKSIZE} both inclusive. For a negative 369 * <tt>inputLength</tt> this method returns <tt>MAX_BLOCKSIZE</tt> 370 * always. 371 * 372 * @param inputLength 373 * The length of the data which will be compressed by 374 * <tt>CBZip2OutputStream</tt>. 375 */ 376 public static int chooseBlockSize(long inputLength) { 377 return (inputLength > 0) ? (int) Math 378 .min((inputLength / 132000) + 1, 9) : MAX_BLOCKSIZE; 379 } 380 381 /** 382 * Constructs a new <tt>CBZip2OutputStream</tt> with a blocksize of 900k. 383 * 384 * @param out 385 * the destination stream. 386 * 387 * @throws IOException 388 * if an I/O error occurs in the specified stream. 389 * @throws NullPointerException 390 * if <code>out == null</code>. 391 */ 392 public BZip2CompressorOutputStream(final OutputStream out) 393 throws IOException { 394 this(out, MAX_BLOCKSIZE); 395 } 396 397 /** 398 * Constructs a new <tt>CBZip2OutputStream</tt> with specified blocksize. 399 * 400 * @param out 401 * the destination stream. 402 * @param blockSize 403 * the blockSize as 100k units. 404 * 405 * @throws IOException 406 * if an I/O error occurs in the specified stream. 407 * @throws IllegalArgumentException 408 * if <code>(blockSize < 1) || (blockSize > 9)</code>. 409 * @throws NullPointerException 410 * if <code>out == null</code>. 411 * 412 * @see #MIN_BLOCKSIZE 413 * @see #MAX_BLOCKSIZE 414 */ 415 public BZip2CompressorOutputStream(final OutputStream out, 416 final int blockSize) 417 throws IOException { 418 super(); 419 420 if (blockSize < 1) { 421 throw new IllegalArgumentException("blockSize(" + blockSize 422 + ") < 1"); 423 } 424 if (blockSize > 9) { 425 throw new IllegalArgumentException("blockSize(" + blockSize 426 + ") > 9"); 427 } 428 429 this.blockSize100k = blockSize; 430 this.out = out; 431 init(); 432 } 433 434 /** {@inheritDoc} */ 435 public void write(final int b) throws IOException { 436 if (this.out != null) { 437 write0(b); 438 } else { 439 throw new IOException("closed"); 440 } 441 } 442 443 private void writeRun() throws IOException { 444 final int lastShadow = this.last; 445 446 if (lastShadow < this.allowableBlockSize) { 447 final int currentCharShadow = this.currentChar; 448 final Data dataShadow = this.data; 449 dataShadow.inUse[currentCharShadow] = true; 450 final byte ch = (byte) currentCharShadow; 451 452 int runLengthShadow = this.runLength; 453 this.crc.updateCRC(currentCharShadow, runLengthShadow); 454 455 switch (runLengthShadow) { 456 case 1: 457 dataShadow.block[lastShadow + 2] = ch; 458 this.last = lastShadow + 1; 459 break; 460 461 case 2: 462 dataShadow.block[lastShadow + 2] = ch; 463 dataShadow.block[lastShadow + 3] = ch; 464 this.last = lastShadow + 2; 465 break; 466 467 case 3: { 468 final byte[] block = dataShadow.block; 469 block[lastShadow + 2] = ch; 470 block[lastShadow + 3] = ch; 471 block[lastShadow + 4] = ch; 472 this.last = lastShadow + 3; 473 } 474 break; 475 476 default: { 477 runLengthShadow -= 4; 478 dataShadow.inUse[runLengthShadow] = true; 479 final byte[] block = dataShadow.block; 480 block[lastShadow + 2] = ch; 481 block[lastShadow + 3] = ch; 482 block[lastShadow + 4] = ch; 483 block[lastShadow + 5] = ch; 484 block[lastShadow + 6] = (byte) runLengthShadow; 485 this.last = lastShadow + 5; 486 } 487 break; 488 489 } 490 } else { 491 endBlock(); 492 initBlock(); 493 writeRun(); 494 } 495 } 496 497 /** 498 * Overriden to close the stream. 499 */ 500 protected void finalize() throws Throwable { 501 finish(); 502 super.finalize(); 503 } 504 505 506 public void finish() throws IOException { 507 if (out != null) { 508 try { 509 if (this.runLength > 0) { 510 writeRun(); 511 } 512 this.currentChar = -1; 513 endBlock(); 514 endCompression(); 515 } finally { 516 this.out = null; 517 this.data = null; 518 } 519 } 520 } 521 522 public void close() throws IOException { 523 if (out != null) { 524 OutputStream outShadow = this.out; 525 finish(); 526 outShadow.close(); 527 } 528 } 529 530 public void flush() throws IOException { 531 OutputStream outShadow = this.out; 532 if (outShadow != null) { 533 outShadow.flush(); 534 } 535 } 536 537 /** 538 * Writes magic bytes like BZ on the first position of the stream 539 * and bytes indiciating the file-format, which is 540 * huffmanised, followed by a digit indicating blockSize100k. 541 * @throws IOException if the magic bytes could not been written 542 */ 543 private void init() throws IOException { 544 bsPutUByte('B'); 545 bsPutUByte('Z'); 546 547 this.data = new Data(this.blockSize100k); 548 549 // huffmanised magic bytes 550 bsPutUByte('h'); 551 bsPutUByte('0' + this.blockSize100k); 552 553 this.combinedCRC = 0; 554 initBlock(); 555 } 556 557 private void initBlock() { 558 // blockNo++; 559 this.crc.initialiseCRC(); 560 this.last = -1; 561 // ch = 0; 562 563 boolean[] inUse = this.data.inUse; 564 for (int i = 256; --i >= 0;) { 565 inUse[i] = false; 566 } 567 568 /* 20 is just a paranoia constant */ 569 this.allowableBlockSize = (this.blockSize100k * BZip2Constants.BASEBLOCKSIZE) - 20; 570 } 571 572 private void endBlock() throws IOException { 573 this.blockCRC = this.crc.getFinalCRC(); 574 this.combinedCRC = (this.combinedCRC << 1) | (this.combinedCRC >>> 31); 575 this.combinedCRC ^= this.blockCRC; 576 577 // empty block at end of file 578 if (this.last == -1) { 579 return; 580 } 581 582 /* sort the block and establish posn of original string */ 583 blockSort(); 584 585 /* 586 * A 6-byte block header, the value chosen arbitrarily as 0x314159265359 587 * :-). A 32 bit value does not really give a strong enough guarantee 588 * that the value will not appear by chance in the compressed 589 * datastream. Worst-case probability of this event, for a 900k block, 590 * is about 2.0e-3 for 32 bits, 1.0e-5 for 40 bits and 4.0e-8 for 48 591 * bits. For a compressed file of size 100Gb -- about 100000 blocks -- 592 * only a 48-bit marker will do. NB: normal compression/ decompression 593 * donot rely on these statistical properties. They are only important 594 * when trying to recover blocks from damaged files. 595 */ 596 bsPutUByte(0x31); 597 bsPutUByte(0x41); 598 bsPutUByte(0x59); 599 bsPutUByte(0x26); 600 bsPutUByte(0x53); 601 bsPutUByte(0x59); 602 603 /* Now the block's CRC, so it is in a known place. */ 604 bsPutInt(this.blockCRC); 605 606 /* Now a single bit indicating randomisation. */ 607 if (this.blockRandomised) { 608 bsW(1, 1); 609 } else { 610 bsW(1, 0); 611 } 612 613 /* Finally, block's contents proper. */ 614 moveToFrontCodeAndSend(); 615 } 616 617 private void endCompression() throws IOException { 618 /* 619 * Now another magic 48-bit number, 0x177245385090, to indicate the end 620 * of the last block. (sqrt(pi), if you want to know. I did want to use 621 * e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me 622 * to feel statistically comfortable. Call me paranoid.) 623 */ 624 bsPutUByte(0x17); 625 bsPutUByte(0x72); 626 bsPutUByte(0x45); 627 bsPutUByte(0x38); 628 bsPutUByte(0x50); 629 bsPutUByte(0x90); 630 631 bsPutInt(this.combinedCRC); 632 bsFinishedWithStream(); 633 } 634 635 /** 636 * Returns the blocksize parameter specified at construction time. 637 */ 638 public final int getBlockSize() { 639 return this.blockSize100k; 640 } 641 642 public void write(final byte[] buf, int offs, final int len) 643 throws IOException { 644 if (offs < 0) { 645 throw new IndexOutOfBoundsException("offs(" + offs + ") < 0."); 646 } 647 if (len < 0) { 648 throw new IndexOutOfBoundsException("len(" + len + ") < 0."); 649 } 650 if (offs + len > buf.length) { 651 throw new IndexOutOfBoundsException("offs(" + offs + ") + len(" 652 + len + ") > buf.length(" 653 + buf.length + ")."); 654 } 655 if (this.out == null) { 656 throw new IOException("stream closed"); 657 } 658 659 for (int hi = offs + len; offs < hi;) { 660 write0(buf[offs++]); 661 } 662 } 663 664 private void write0(int b) throws IOException { 665 if (this.currentChar != -1) { 666 b &= 0xff; 667 if (this.currentChar == b) { 668 if (++this.runLength > 254) { 669 writeRun(); 670 this.currentChar = -1; 671 this.runLength = 0; 672 } 673 // else nothing to do 674 } else { 675 writeRun(); 676 this.runLength = 1; 677 this.currentChar = b; 678 } 679 } else { 680 this.currentChar = b & 0xff; 681 this.runLength++; 682 } 683 } 684 685 private static void hbAssignCodes(final int[] code, final byte[] length, 686 final int minLen, final int maxLen, 687 final int alphaSize) { 688 int vec = 0; 689 for (int n = minLen; n <= maxLen; n++) { 690 for (int i = 0; i < alphaSize; i++) { 691 if ((length[i] & 0xff) == n) { 692 code[i] = vec; 693 vec++; 694 } 695 } 696 vec <<= 1; 697 } 698 } 699 700 private void bsFinishedWithStream() throws IOException { 701 while (this.bsLive > 0) { 702 int ch = this.bsBuff >> 24; 703 this.out.write(ch); // write 8-bit 704 this.bsBuff <<= 8; 705 this.bsLive -= 8; 706 } 707 } 708 709 private void bsW(final int n, final int v) throws IOException { 710 final OutputStream outShadow = this.out; 711 int bsLiveShadow = this.bsLive; 712 int bsBuffShadow = this.bsBuff; 713 714 while (bsLiveShadow >= 8) { 715 outShadow.write(bsBuffShadow >> 24); // write 8-bit 716 bsBuffShadow <<= 8; 717 bsLiveShadow -= 8; 718 } 719 720 this.bsBuff = bsBuffShadow | (v << (32 - bsLiveShadow - n)); 721 this.bsLive = bsLiveShadow + n; 722 } 723 724 private void bsPutUByte(final int c) throws IOException { 725 bsW(8, c); 726 } 727 728 private void bsPutInt(final int u) throws IOException { 729 bsW(8, (u >> 24) & 0xff); 730 bsW(8, (u >> 16) & 0xff); 731 bsW(8, (u >> 8) & 0xff); 732 bsW(8, u & 0xff); 733 } 734 735 private void sendMTFValues() throws IOException { 736 final byte[][] len = this.data.sendMTFValues_len; 737 final int alphaSize = this.nInUse + 2; 738 739 for (int t = N_GROUPS; --t >= 0;) { 740 byte[] len_t = len[t]; 741 for (int v = alphaSize; --v >= 0;) { 742 len_t[v] = GREATER_ICOST; 743 } 744 } 745 746 /* Decide how many coding tables to use */ 747 // assert (this.nMTF > 0) : this.nMTF; 748 final int nGroups = (this.nMTF < 200) ? 2 : (this.nMTF < 600) ? 3 749 : (this.nMTF < 1200) ? 4 : (this.nMTF < 2400) ? 5 : 6; 750 751 /* Generate an initial set of coding tables */ 752 sendMTFValues0(nGroups, alphaSize); 753 754 /* 755 * Iterate up to N_ITERS times to improve the tables. 756 */ 757 final int nSelectors = sendMTFValues1(nGroups, alphaSize); 758 759 /* Compute MTF values for the selectors. */ 760 sendMTFValues2(nGroups, nSelectors); 761 762 /* Assign actual codes for the tables. */ 763 sendMTFValues3(nGroups, alphaSize); 764 765 /* Transmit the mapping table. */ 766 sendMTFValues4(); 767 768 /* Now the selectors. */ 769 sendMTFValues5(nGroups, nSelectors); 770 771 /* Now the coding tables. */ 772 sendMTFValues6(nGroups, alphaSize); 773 774 /* And finally, the block data proper */ 775 sendMTFValues7(); 776 } 777 778 private void sendMTFValues0(final int nGroups, final int alphaSize) { 779 final byte[][] len = this.data.sendMTFValues_len; 780 final int[] mtfFreq = this.data.mtfFreq; 781 782 int remF = this.nMTF; 783 int gs = 0; 784 785 for (int nPart = nGroups; nPart > 0; nPart--) { 786 final int tFreq = remF / nPart; 787 int ge = gs - 1; 788 int aFreq = 0; 789 790 for (final int a = alphaSize - 1; (aFreq < tFreq) && (ge < a);) { 791 aFreq += mtfFreq[++ge]; 792 } 793 794 if ((ge > gs) && (nPart != nGroups) && (nPart != 1) 795 && (((nGroups - nPart) & 1) != 0)) { 796 aFreq -= mtfFreq[ge--]; 797 } 798 799 final byte[] len_np = len[nPart - 1]; 800 for (int v = alphaSize; --v >= 0;) { 801 if ((v >= gs) && (v <= ge)) { 802 len_np[v] = LESSER_ICOST; 803 } else { 804 len_np[v] = GREATER_ICOST; 805 } 806 } 807 808 gs = ge + 1; 809 remF -= aFreq; 810 } 811 } 812 813 private int sendMTFValues1(final int nGroups, final int alphaSize) { 814 final Data dataShadow = this.data; 815 final int[][] rfreq = dataShadow.sendMTFValues_rfreq; 816 final int[] fave = dataShadow.sendMTFValues_fave; 817 final short[] cost = dataShadow.sendMTFValues_cost; 818 final char[] sfmap = dataShadow.sfmap; 819 final byte[] selector = dataShadow.selector; 820 final byte[][] len = dataShadow.sendMTFValues_len; 821 final byte[] len_0 = len[0]; 822 final byte[] len_1 = len[1]; 823 final byte[] len_2 = len[2]; 824 final byte[] len_3 = len[3]; 825 final byte[] len_4 = len[4]; 826 final byte[] len_5 = len[5]; 827 final int nMTFShadow = this.nMTF; 828 829 int nSelectors = 0; 830 831 for (int iter = 0; iter < N_ITERS; iter++) { 832 for (int t = nGroups; --t >= 0;) { 833 fave[t] = 0; 834 int[] rfreqt = rfreq[t]; 835 for (int i = alphaSize; --i >= 0;) { 836 rfreqt[i] = 0; 837 } 838 } 839 840 nSelectors = 0; 841 842 for (int gs = 0; gs < this.nMTF;) { 843 /* Set group start & end marks. */ 844 845 /* 846 * Calculate the cost of this group as coded by each of the 847 * coding tables. 848 */ 849 850 final int ge = Math.min(gs + G_SIZE - 1, nMTFShadow - 1); 851 852 if (nGroups == N_GROUPS) { 853 // unrolled version of the else-block 854 855 short cost0 = 0; 856 short cost1 = 0; 857 short cost2 = 0; 858 short cost3 = 0; 859 short cost4 = 0; 860 short cost5 = 0; 861 862 for (int i = gs; i <= ge; i++) { 863 final int icv = sfmap[i]; 864 cost0 += len_0[icv] & 0xff; 865 cost1 += len_1[icv] & 0xff; 866 cost2 += len_2[icv] & 0xff; 867 cost3 += len_3[icv] & 0xff; 868 cost4 += len_4[icv] & 0xff; 869 cost5 += len_5[icv] & 0xff; 870 } 871 872 cost[0] = cost0; 873 cost[1] = cost1; 874 cost[2] = cost2; 875 cost[3] = cost3; 876 cost[4] = cost4; 877 cost[5] = cost5; 878 879 } else { 880 for (int t = nGroups; --t >= 0;) { 881 cost[t] = 0; 882 } 883 884 for (int i = gs; i <= ge; i++) { 885 final int icv = sfmap[i]; 886 for (int t = nGroups; --t >= 0;) { 887 cost[t] += len[t][icv] & 0xff; 888 } 889 } 890 } 891 892 /* 893 * Find the coding table which is best for this group, and 894 * record its identity in the selector table. 895 */ 896 int bt = -1; 897 for (int t = nGroups, bc = 999999999; --t >= 0;) { 898 final int cost_t = cost[t]; 899 if (cost_t < bc) { 900 bc = cost_t; 901 bt = t; 902 } 903 } 904 905 fave[bt]++; 906 selector[nSelectors] = (byte) bt; 907 nSelectors++; 908 909 /* 910 * Increment the symbol frequencies for the selected table. 911 */ 912 final int[] rfreq_bt = rfreq[bt]; 913 for (int i = gs; i <= ge; i++) { 914 rfreq_bt[sfmap[i]]++; 915 } 916 917 gs = ge + 1; 918 } 919 920 /* 921 * Recompute the tables based on the accumulated frequencies. 922 */ 923 for (int t = 0; t < nGroups; t++) { 924 hbMakeCodeLengths(len[t], rfreq[t], this.data, alphaSize, 20); 925 } 926 } 927 928 return nSelectors; 929 } 930 931 private void sendMTFValues2(final int nGroups, final int nSelectors) { 932 // assert (nGroups < 8) : nGroups; 933 934 final Data dataShadow = this.data; 935 byte[] pos = dataShadow.sendMTFValues2_pos; 936 937 for (int i = nGroups; --i >= 0;) { 938 pos[i] = (byte) i; 939 } 940 941 for (int i = 0; i < nSelectors; i++) { 942 final byte ll_i = dataShadow.selector[i]; 943 byte tmp = pos[0]; 944 int j = 0; 945 946 while (ll_i != tmp) { 947 j++; 948 byte tmp2 = tmp; 949 tmp = pos[j]; 950 pos[j] = tmp2; 951 } 952 953 pos[0] = tmp; 954 dataShadow.selectorMtf[i] = (byte) j; 955 } 956 } 957 958 private void sendMTFValues3(final int nGroups, final int alphaSize) { 959 int[][] code = this.data.sendMTFValues_code; 960 byte[][] len = this.data.sendMTFValues_len; 961 962 for (int t = 0; t < nGroups; t++) { 963 int minLen = 32; 964 int maxLen = 0; 965 final byte[] len_t = len[t]; 966 for (int i = alphaSize; --i >= 0;) { 967 final int l = len_t[i] & 0xff; 968 if (l > maxLen) { 969 maxLen = l; 970 } 971 if (l < minLen) { 972 minLen = l; 973 } 974 } 975 976 // assert (maxLen <= 20) : maxLen; 977 // assert (minLen >= 1) : minLen; 978 979 hbAssignCodes(code[t], len[t], minLen, maxLen, alphaSize); 980 } 981 } 982 983 private void sendMTFValues4() throws IOException { 984 final boolean[] inUse = this.data.inUse; 985 final boolean[] inUse16 = this.data.sentMTFValues4_inUse16; 986 987 for (int i = 16; --i >= 0;) { 988 inUse16[i] = false; 989 final int i16 = i * 16; 990 for (int j = 16; --j >= 0;) { 991 if (inUse[i16 + j]) { 992 inUse16[i] = true; 993 } 994 } 995 } 996 997 for (int i = 0; i < 16; i++) { 998 bsW(1, inUse16[i] ? 1 : 0); 999 } 1000 1001 final OutputStream outShadow = this.out; 1002 int bsLiveShadow = this.bsLive; 1003 int bsBuffShadow = this.bsBuff; 1004 1005 for (int i = 0; i < 16; i++) { 1006 if (inUse16[i]) { 1007 final int i16 = i * 16; 1008 for (int j = 0; j < 16; j++) { 1009 // inlined: bsW(1, inUse[i16 + j] ? 1 : 0); 1010 while (bsLiveShadow >= 8) { 1011 outShadow.write(bsBuffShadow >> 24); // write 8-bit 1012 bsBuffShadow <<= 8; 1013 bsLiveShadow -= 8; 1014 } 1015 if (inUse[i16 + j]) { 1016 bsBuffShadow |= 1 << (32 - bsLiveShadow - 1); 1017 } 1018 bsLiveShadow++; 1019 } 1020 } 1021 } 1022 1023 this.bsBuff = bsBuffShadow; 1024 this.bsLive = bsLiveShadow; 1025 } 1026 1027 private void sendMTFValues5(final int nGroups, final int nSelectors) 1028 throws IOException { 1029 bsW(3, nGroups); 1030 bsW(15, nSelectors); 1031 1032 final OutputStream outShadow = this.out; 1033 final byte[] selectorMtf = this.data.selectorMtf; 1034 1035 int bsLiveShadow = this.bsLive; 1036 int bsBuffShadow = this.bsBuff; 1037 1038 for (int i = 0; i < nSelectors; i++) { 1039 for (int j = 0, hj = selectorMtf[i] & 0xff; j < hj; j++) { 1040 // inlined: bsW(1, 1); 1041 while (bsLiveShadow >= 8) { 1042 outShadow.write(bsBuffShadow >> 24); 1043 bsBuffShadow <<= 8; 1044 bsLiveShadow -= 8; 1045 } 1046 bsBuffShadow |= 1 << (32 - bsLiveShadow - 1); 1047 bsLiveShadow++; 1048 } 1049 1050 // inlined: bsW(1, 0); 1051 while (bsLiveShadow >= 8) { 1052 outShadow.write(bsBuffShadow >> 24); 1053 bsBuffShadow <<= 8; 1054 bsLiveShadow -= 8; 1055 } 1056 // bsBuffShadow |= 0 << (32 - bsLiveShadow - 1); 1057 bsLiveShadow++; 1058 } 1059 1060 this.bsBuff = bsBuffShadow; 1061 this.bsLive = bsLiveShadow; 1062 } 1063 1064 private void sendMTFValues6(final int nGroups, final int alphaSize) 1065 throws IOException { 1066 final byte[][] len = this.data.sendMTFValues_len; 1067 final OutputStream outShadow = this.out; 1068 1069 int bsLiveShadow = this.bsLive; 1070 int bsBuffShadow = this.bsBuff; 1071 1072 for (int t = 0; t < nGroups; t++) { 1073 byte[] len_t = len[t]; 1074 int curr = len_t[0] & 0xff; 1075 1076 // inlined: bsW(5, curr); 1077 while (bsLiveShadow >= 8) { 1078 outShadow.write(bsBuffShadow >> 24); // write 8-bit 1079 bsBuffShadow <<= 8; 1080 bsLiveShadow -= 8; 1081 } 1082 bsBuffShadow |= curr << (32 - bsLiveShadow - 5); 1083 bsLiveShadow += 5; 1084 1085 for (int i = 0; i < alphaSize; i++) { 1086 int lti = len_t[i] & 0xff; 1087 while (curr < lti) { 1088 // inlined: bsW(2, 2); 1089 while (bsLiveShadow >= 8) { 1090 outShadow.write(bsBuffShadow >> 24); // write 8-bit 1091 bsBuffShadow <<= 8; 1092 bsLiveShadow -= 8; 1093 } 1094 bsBuffShadow |= 2 << (32 - bsLiveShadow - 2); 1095 bsLiveShadow += 2; 1096 1097 curr++; /* 10 */ 1098 } 1099 1100 while (curr > lti) { 1101 // inlined: bsW(2, 3); 1102 while (bsLiveShadow >= 8) { 1103 outShadow.write(bsBuffShadow >> 24); // write 8-bit 1104 bsBuffShadow <<= 8; 1105 bsLiveShadow -= 8; 1106 } 1107 bsBuffShadow |= 3 << (32 - bsLiveShadow - 2); 1108 bsLiveShadow += 2; 1109 1110 curr--; /* 11 */ 1111 } 1112 1113 // inlined: bsW(1, 0); 1114 while (bsLiveShadow >= 8) { 1115 outShadow.write(bsBuffShadow >> 24); // write 8-bit 1116 bsBuffShadow <<= 8; 1117 bsLiveShadow -= 8; 1118 } 1119 // bsBuffShadow |= 0 << (32 - bsLiveShadow - 1); 1120 bsLiveShadow++; 1121 } 1122 } 1123 1124 this.bsBuff = bsBuffShadow; 1125 this.bsLive = bsLiveShadow; 1126 } 1127 1128 private void sendMTFValues7() throws IOException { 1129 final Data dataShadow = this.data; 1130 final byte[][] len = dataShadow.sendMTFValues_len; 1131 final int[][] code = dataShadow.sendMTFValues_code; 1132 final OutputStream outShadow = this.out; 1133 final byte[] selector = dataShadow.selector; 1134 final char[] sfmap = dataShadow.sfmap; 1135 final int nMTFShadow = this.nMTF; 1136 1137 int selCtr = 0; 1138 1139 int bsLiveShadow = this.bsLive; 1140 int bsBuffShadow = this.bsBuff; 1141 1142 for (int gs = 0; gs < nMTFShadow;) { 1143 final int ge = Math.min(gs + G_SIZE - 1, nMTFShadow - 1); 1144 final int selector_selCtr = selector[selCtr] & 0xff; 1145 final int[] code_selCtr = code[selector_selCtr]; 1146 final byte[] len_selCtr = len[selector_selCtr]; 1147 1148 while (gs <= ge) { 1149 final int sfmap_i = sfmap[gs]; 1150 1151 // 1152 // inlined: bsW(len_selCtr[sfmap_i] & 0xff, 1153 // code_selCtr[sfmap_i]); 1154 // 1155 while (bsLiveShadow >= 8) { 1156 outShadow.write(bsBuffShadow >> 24); 1157 bsBuffShadow <<= 8; 1158 bsLiveShadow -= 8; 1159 } 1160 final int n = len_selCtr[sfmap_i] & 0xFF; 1161 bsBuffShadow |= code_selCtr[sfmap_i] << (32 - bsLiveShadow - n); 1162 bsLiveShadow += n; 1163 1164 gs++; 1165 } 1166 1167 gs = ge + 1; 1168 selCtr++; 1169 } 1170 1171 this.bsBuff = bsBuffShadow; 1172 this.bsLive = bsLiveShadow; 1173 } 1174 1175 private void moveToFrontCodeAndSend() throws IOException { 1176 bsW(24, this.origPtr); 1177 generateMTFValues(); 1178 sendMTFValues(); 1179 } 1180 1181 /** 1182 * This is the most hammered method of this class. 1183 * 1184 * <p> 1185 * This is the version using unrolled loops. Normally I never use such ones 1186 * in Java code. The unrolling has shown a noticable performance improvement 1187 * on JRE 1.4.2 (Linux i586 / HotSpot Client). Of course it depends on the 1188 * JIT compiler of the vm. 1189 * </p> 1190 */ 1191 private boolean mainSimpleSort(final Data dataShadow, final int lo, 1192 final int hi, final int d) { 1193 final int bigN = hi - lo + 1; 1194 if (bigN < 2) { 1195 return this.firstAttempt && (this.workDone > this.workLimit); 1196 } 1197 1198 int hp = 0; 1199 while (INCS[hp] < bigN) { 1200 hp++; 1201 } 1202 1203 final int[] fmap = dataShadow.fmap; 1204 final char[] quadrant = dataShadow.quadrant; 1205 final byte[] block = dataShadow.block; 1206 final int lastShadow = this.last; 1207 final int lastPlus1 = lastShadow + 1; 1208 final boolean firstAttemptShadow = this.firstAttempt; 1209 final int workLimitShadow = this.workLimit; 1210 int workDoneShadow = this.workDone; 1211 1212 // Following block contains unrolled code which could be shortened by 1213 // coding it in additional loops. 1214 1215 HP: while (--hp >= 0) { 1216 final int h = INCS[hp]; 1217 final int mj = lo + h - 1; 1218 1219 for (int i = lo + h; i <= hi;) { 1220 // copy 1221 for (int k = 3; (i <= hi) && (--k >= 0); i++) { 1222 final int v = fmap[i]; 1223 final int vd = v + d; 1224 int j = i; 1225 1226 // for (int a; 1227 // (j > mj) && mainGtU((a = fmap[j - h]) + d, vd, 1228 // block, quadrant, lastShadow); 1229 // j -= h) { 1230 // fmap[j] = a; 1231 // } 1232 // 1233 // unrolled version: 1234 1235 // start inline mainGTU 1236 boolean onceRunned = false; 1237 int a = 0; 1238 1239 HAMMER: while (true) { 1240 if (onceRunned) { 1241 fmap[j] = a; 1242 if ((j -= h) <= mj) { 1243 break HAMMER; 1244 } 1245 } else { 1246 onceRunned = true; 1247 } 1248 1249 a = fmap[j - h]; 1250 int i1 = a + d; 1251 int i2 = vd; 1252 1253 // following could be done in a loop, but 1254 // unrolled it for performance: 1255 if (block[i1 + 1] == block[i2 + 1]) { 1256 if (block[i1 + 2] == block[i2 + 2]) { 1257 if (block[i1 + 3] == block[i2 + 3]) { 1258 if (block[i1 + 4] == block[i2 + 4]) { 1259 if (block[i1 + 5] == block[i2 + 5]) { 1260 if (block[(i1 += 6)] == block[(i2 += 6)]) { 1261 int x = lastShadow; 1262 X: while (x > 0) { 1263 x -= 4; 1264 1265 if (block[i1 + 1] == block[i2 + 1]) { 1266 if (quadrant[i1] == quadrant[i2]) { 1267 if (block[i1 + 2] == block[i2 + 2]) { 1268 if (quadrant[i1 + 1] == quadrant[i2 + 1]) { 1269 if (block[i1 + 3] == block[i2 + 3]) { 1270 if (quadrant[i1 + 2] == quadrant[i2 + 2]) { 1271 if (block[i1 + 4] == block[i2 + 4]) { 1272 if (quadrant[i1 + 3] == quadrant[i2 + 3]) { 1273 if ((i1 += 4) >= lastPlus1) { 1274 i1 -= lastPlus1; 1275 } 1276 if ((i2 += 4) >= lastPlus1) { 1277 i2 -= lastPlus1; 1278 } 1279 workDoneShadow++; 1280 continue X; 1281 } else if ((quadrant[i1 + 3] > quadrant[i2 + 3])) { 1282 continue HAMMER; 1283 } else { 1284 break HAMMER; 1285 } 1286 } else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) { 1287 continue HAMMER; 1288 } else { 1289 break HAMMER; 1290 } 1291 } else if ((quadrant[i1 + 2] > quadrant[i2 + 2])) { 1292 continue HAMMER; 1293 } else { 1294 break HAMMER; 1295 } 1296 } else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) { 1297 continue HAMMER; 1298 } else { 1299 break HAMMER; 1300 } 1301 } else if ((quadrant[i1 + 1] > quadrant[i2 + 1])) { 1302 continue HAMMER; 1303 } else { 1304 break HAMMER; 1305 } 1306 } else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) { 1307 continue HAMMER; 1308 } else { 1309 break HAMMER; 1310 } 1311 } else if ((quadrant[i1] > quadrant[i2])) { 1312 continue HAMMER; 1313 } else { 1314 break HAMMER; 1315 } 1316 } else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) { 1317 continue HAMMER; 1318 } else { 1319 break HAMMER; 1320 } 1321 1322 } 1323 break HAMMER; 1324 } // while x > 0 1325 else { 1326 if ((block[i1] & 0xff) > (block[i2] & 0xff)) { 1327 continue HAMMER; 1328 } else { 1329 break HAMMER; 1330 } 1331 } 1332 } else if ((block[i1 + 5] & 0xff) > (block[i2 + 5] & 0xff)) { 1333 continue HAMMER; 1334 } else { 1335 break HAMMER; 1336 } 1337 } else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) { 1338 continue HAMMER; 1339 } else { 1340 break HAMMER; 1341 } 1342 } else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) { 1343 continue HAMMER; 1344 } else { 1345 break HAMMER; 1346 } 1347 } else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) { 1348 continue HAMMER; 1349 } else { 1350 break HAMMER; 1351 } 1352 } else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) { 1353 continue HAMMER; 1354 } else { 1355 break HAMMER; 1356 } 1357 1358 } // HAMMER 1359 // end inline mainGTU 1360 1361 fmap[j] = v; 1362 } 1363 1364 if (firstAttemptShadow && (i <= hi) 1365 && (workDoneShadow > workLimitShadow)) { 1366 break HP; 1367 } 1368 } 1369 } 1370 1371 this.workDone = workDoneShadow; 1372 return firstAttemptShadow && (workDoneShadow > workLimitShadow); 1373 } 1374 1375 private static void vswap(int[] fmap, int p1, int p2, int n) { 1376 n += p1; 1377 while (p1 < n) { 1378 int t = fmap[p1]; 1379 fmap[p1++] = fmap[p2]; 1380 fmap[p2++] = t; 1381 } 1382 } 1383 1384 private static byte med3(byte a, byte b, byte c) { 1385 return (a < b) ? (b < c ? b : a < c ? c : a) : (b > c ? b : a > c ? c 1386 : a); 1387 } 1388 1389 private void blockSort() { 1390 this.workLimit = WORK_FACTOR * this.last; 1391 this.workDone = 0; 1392 this.blockRandomised = false; 1393 this.firstAttempt = true; 1394 mainSort(); 1395 1396 if (this.firstAttempt && (this.workDone > this.workLimit)) { 1397 randomiseBlock(); 1398 this.workLimit = this.workDone = 0; 1399 this.firstAttempt = false; 1400 mainSort(); 1401 } 1402 1403 int[] fmap = this.data.fmap; 1404 this.origPtr = -1; 1405 for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) { 1406 if (fmap[i] == 0) { 1407 this.origPtr = i; 1408 break; 1409 } 1410 } 1411 1412 // assert (this.origPtr != -1) : this.origPtr; 1413 } 1414 1415 /** 1416 * Method "mainQSort3", file "blocksort.c", BZip2 1.0.2 1417 */ 1418 private void mainQSort3(final Data dataShadow, final int loSt, 1419 final int hiSt, final int dSt) { 1420 final int[] stack_ll = dataShadow.stack_ll; 1421 final int[] stack_hh = dataShadow.stack_hh; 1422 final int[] stack_dd = dataShadow.stack_dd; 1423 final int[] fmap = dataShadow.fmap; 1424 final byte[] block = dataShadow.block; 1425 1426 stack_ll[0] = loSt; 1427 stack_hh[0] = hiSt; 1428 stack_dd[0] = dSt; 1429 1430 for (int sp = 1; --sp >= 0;) { 1431 final int lo = stack_ll[sp]; 1432 final int hi = stack_hh[sp]; 1433 final int d = stack_dd[sp]; 1434 1435 if ((hi - lo < SMALL_THRESH) || (d > DEPTH_THRESH)) { 1436 if (mainSimpleSort(dataShadow, lo, hi, d)) { 1437 return; 1438 } 1439 } else { 1440 final int d1 = d + 1; 1441 final int med = med3(block[fmap[lo] + d1], 1442 block[fmap[hi] + d1], block[fmap[(lo + hi) >>> 1] + d1]) & 0xff; 1443 1444 int unLo = lo; 1445 int unHi = hi; 1446 int ltLo = lo; 1447 int gtHi = hi; 1448 1449 while (true) { 1450 while (unLo <= unHi) { 1451 final int n = (block[fmap[unLo] + d1] & 0xff) 1452 - med; 1453 if (n == 0) { 1454 final int temp = fmap[unLo]; 1455 fmap[unLo++] = fmap[ltLo]; 1456 fmap[ltLo++] = temp; 1457 } else if (n < 0) { 1458 unLo++; 1459 } else { 1460 break; 1461 } 1462 } 1463 1464 while (unLo <= unHi) { 1465 final int n = (block[fmap[unHi] + d1] & 0xff) 1466 - med; 1467 if (n == 0) { 1468 final int temp = fmap[unHi]; 1469 fmap[unHi--] = fmap[gtHi]; 1470 fmap[gtHi--] = temp; 1471 } else if (n > 0) { 1472 unHi--; 1473 } else { 1474 break; 1475 } 1476 } 1477 1478 if (unLo <= unHi) { 1479 final int temp = fmap[unLo]; 1480 fmap[unLo++] = fmap[unHi]; 1481 fmap[unHi--] = temp; 1482 } else { 1483 break; 1484 } 1485 } 1486 1487 if (gtHi < ltLo) { 1488 stack_ll[sp] = lo; 1489 stack_hh[sp] = hi; 1490 stack_dd[sp] = d1; 1491 sp++; 1492 } else { 1493 int n = ((ltLo - lo) < (unLo - ltLo)) ? (ltLo - lo) 1494 : (unLo - ltLo); 1495 vswap(fmap, lo, unLo - n, n); 1496 int m = ((hi - gtHi) < (gtHi - unHi)) ? (hi - gtHi) 1497 : (gtHi - unHi); 1498 vswap(fmap, unLo, hi - m + 1, m); 1499 1500 n = lo + unLo - ltLo - 1; 1501 m = hi - (gtHi - unHi) + 1; 1502 1503 stack_ll[sp] = lo; 1504 stack_hh[sp] = n; 1505 stack_dd[sp] = d; 1506 sp++; 1507 1508 stack_ll[sp] = n + 1; 1509 stack_hh[sp] = m - 1; 1510 stack_dd[sp] = d1; 1511 sp++; 1512 1513 stack_ll[sp] = m; 1514 stack_hh[sp] = hi; 1515 stack_dd[sp] = d; 1516 sp++; 1517 } 1518 } 1519 } 1520 } 1521 1522 private void mainSort() { 1523 final Data dataShadow = this.data; 1524 final int[] runningOrder = dataShadow.mainSort_runningOrder; 1525 final int[] copy = dataShadow.mainSort_copy; 1526 final boolean[] bigDone = dataShadow.mainSort_bigDone; 1527 final int[] ftab = dataShadow.ftab; 1528 final byte[] block = dataShadow.block; 1529 final int[] fmap = dataShadow.fmap; 1530 final char[] quadrant = dataShadow.quadrant; 1531 final int lastShadow = this.last; 1532 final int workLimitShadow = this.workLimit; 1533 final boolean firstAttemptShadow = this.firstAttempt; 1534 1535 // Set up the 2-byte frequency table 1536 for (int i = 65537; --i >= 0;) { 1537 ftab[i] = 0; 1538 } 1539 1540 /* 1541 * In the various block-sized structures, live data runs from 0 to 1542 * last+NUM_OVERSHOOT_BYTES inclusive. First, set up the overshoot area 1543 * for block. 1544 */ 1545 for (int i = 0; i < NUM_OVERSHOOT_BYTES; i++) { 1546 block[lastShadow + i + 2] = block[(i % (lastShadow + 1)) + 1]; 1547 } 1548 for (int i = lastShadow + NUM_OVERSHOOT_BYTES +1; --i >= 0;) { 1549 quadrant[i] = 0; 1550 } 1551 block[0] = block[lastShadow + 1]; 1552 1553 // Complete the initial radix sort: 1554 1555 int c1 = block[0] & 0xff; 1556 for (int i = 0; i <= lastShadow; i++) { 1557 final int c2 = block[i + 1] & 0xff; 1558 ftab[(c1 << 8) + c2]++; 1559 c1 = c2; 1560 } 1561 1562 for (int i = 1; i <= 65536; i++) 1563 ftab[i] += ftab[i - 1]; 1564 1565 c1 = block[1] & 0xff; 1566 for (int i = 0; i < lastShadow; i++) { 1567 final int c2 = block[i + 2] & 0xff; 1568 fmap[--ftab[(c1 << 8) + c2]] = i; 1569 c1 = c2; 1570 } 1571 1572 fmap[--ftab[((block[lastShadow + 1] & 0xff) << 8) + (block[1] & 0xff)]] = lastShadow; 1573 1574 /* 1575 * Now ftab contains the first loc of every small bucket. Calculate the 1576 * running order, from smallest to largest big bucket. 1577 */ 1578 for (int i = 256; --i >= 0;) { 1579 bigDone[i] = false; 1580 runningOrder[i] = i; 1581 } 1582 1583 for (int h = 364; h != 1;) { 1584 h /= 3; 1585 for (int i = h; i <= 255; i++) { 1586 final int vv = runningOrder[i]; 1587 final int a = ftab[(vv + 1) << 8] - ftab[vv << 8]; 1588 final int b = h - 1; 1589 int j = i; 1590 for (int ro = runningOrder[j - h]; (ftab[(ro + 1) << 8] - ftab[ro << 8]) > a; ro = runningOrder[j 1591 - h]) { 1592 runningOrder[j] = ro; 1593 j -= h; 1594 if (j <= b) { 1595 break; 1596 } 1597 } 1598 runningOrder[j] = vv; 1599 } 1600 } 1601 1602 /* 1603 * The main sorting loop. 1604 */ 1605 for (int i = 0; i <= 255; i++) { 1606 /* 1607 * Process big buckets, starting with the least full. 1608 */ 1609 final int ss = runningOrder[i]; 1610 1611 // Step 1: 1612 /* 1613 * Complete the big bucket [ss] by quicksorting any unsorted small 1614 * buckets [ss, j]. Hopefully previous pointer-scanning phases have 1615 * already completed many of the small buckets [ss, j], so we don't 1616 * have to sort them at all. 1617 */ 1618 for (int j = 0; j <= 255; j++) { 1619 final int sb = (ss << 8) + j; 1620 final int ftab_sb = ftab[sb]; 1621 if ((ftab_sb & SETMASK) != SETMASK) { 1622 final int lo = ftab_sb & CLEARMASK; 1623 final int hi = (ftab[sb + 1] & CLEARMASK) - 1; 1624 if (hi > lo) { 1625 mainQSort3(dataShadow, lo, hi, 2); 1626 if (firstAttemptShadow 1627 && (this.workDone > workLimitShadow)) { 1628 return; 1629 } 1630 } 1631 ftab[sb] = ftab_sb | SETMASK; 1632 } 1633 } 1634 1635 // Step 2: 1636 // Now scan this big bucket so as to synthesise the 1637 // sorted order for small buckets [t, ss] for all t != ss. 1638 1639 for (int j = 0; j <= 255; j++) { 1640 copy[j] = ftab[(j << 8) + ss] & CLEARMASK; 1641 } 1642 1643 for (int j = ftab[ss << 8] & CLEARMASK, hj = (ftab[(ss + 1) << 8] & CLEARMASK); j < hj; j++) { 1644 final int fmap_j = fmap[j]; 1645 c1 = block[fmap_j] & 0xff; 1646 if (!bigDone[c1]) { 1647 fmap[copy[c1]] = (fmap_j == 0) ? lastShadow : (fmap_j - 1); 1648 copy[c1]++; 1649 } 1650 } 1651 1652 for (int j = 256; --j >= 0;) 1653 ftab[(j << 8) + ss] |= SETMASK; 1654 1655 // Step 3: 1656 /* 1657 * The ss big bucket is now done. Record this fact, and update the 1658 * quadrant descriptors. Remember to update quadrants in the 1659 * overshoot area too, if necessary. The "if (i < 255)" test merely 1660 * skips this updating for the last bucket processed, since updating 1661 * for the last bucket is pointless. 1662 */ 1663 bigDone[ss] = true; 1664 1665 if (i < 255) { 1666 final int bbStart = ftab[ss << 8] & CLEARMASK; 1667 final int bbSize = (ftab[(ss + 1) << 8] & CLEARMASK) - bbStart; 1668 int shifts = 0; 1669 1670 while ((bbSize >> shifts) > 65534) { 1671 shifts++; 1672 } 1673 1674 for (int j = 0; j < bbSize; j++) { 1675 final int a2update = fmap[bbStart + j]; 1676 final char qVal = (char) (j >> shifts); 1677 quadrant[a2update] = qVal; 1678 if (a2update < NUM_OVERSHOOT_BYTES) { 1679 quadrant[a2update + lastShadow + 1] = qVal; 1680 } 1681 } 1682 } 1683 1684 } 1685 } 1686 1687 private void randomiseBlock() { 1688 final boolean[] inUse = this.data.inUse; 1689 final byte[] block = this.data.block; 1690 final int lastShadow = this.last; 1691 1692 for (int i = 256; --i >= 0;) 1693 inUse[i] = false; 1694 1695 int rNToGo = 0; 1696 int rTPos = 0; 1697 for (int i = 0, j = 1; i <= lastShadow; i = j, j++) { 1698 if (rNToGo == 0) { 1699 rNToGo = (char) Rand.rNums(rTPos); 1700 if (++rTPos == 512) { 1701 rTPos = 0; 1702 } 1703 } 1704 1705 rNToGo--; 1706 block[j] ^= ((rNToGo == 1) ? 1 : 0); 1707 1708 // handle 16 bit signed numbers 1709 inUse[block[j] & 0xff] = true; 1710 } 1711 1712 this.blockRandomised = true; 1713 } 1714 1715 private void generateMTFValues() { 1716 final int lastShadow = this.last; 1717 final Data dataShadow = this.data; 1718 final boolean[] inUse = dataShadow.inUse; 1719 final byte[] block = dataShadow.block; 1720 final int[] fmap = dataShadow.fmap; 1721 final char[] sfmap = dataShadow.sfmap; 1722 final int[] mtfFreq = dataShadow.mtfFreq; 1723 final byte[] unseqToSeq = dataShadow.unseqToSeq; 1724 final byte[] yy = dataShadow.generateMTFValues_yy; 1725 1726 // make maps 1727 int nInUseShadow = 0; 1728 for (int i = 0; i < 256; i++) { 1729 if (inUse[i]) { 1730 unseqToSeq[i] = (byte) nInUseShadow; 1731 nInUseShadow++; 1732 } 1733 } 1734 this.nInUse = nInUseShadow; 1735 1736 final int eob = nInUseShadow + 1; 1737 1738 for (int i = eob; i >= 0; i--) { 1739 mtfFreq[i] = 0; 1740 } 1741 1742 for (int i = nInUseShadow; --i >= 0;) { 1743 yy[i] = (byte) i; 1744 } 1745 1746 int wr = 0; 1747 int zPend = 0; 1748 1749 for (int i = 0; i <= lastShadow; i++) { 1750 final byte ll_i = unseqToSeq[block[fmap[i]] & 0xff]; 1751 byte tmp = yy[0]; 1752 int j = 0; 1753 1754 while (ll_i != tmp) { 1755 j++; 1756 byte tmp2 = tmp; 1757 tmp = yy[j]; 1758 yy[j] = tmp2; 1759 } 1760 yy[0] = tmp; 1761 1762 if (j == 0) { 1763 zPend++; 1764 } else { 1765 if (zPend > 0) { 1766 zPend--; 1767 while (true) { 1768 if ((zPend & 1) == 0) { 1769 sfmap[wr] = RUNA; 1770 wr++; 1771 mtfFreq[RUNA]++; 1772 } else { 1773 sfmap[wr] = RUNB; 1774 wr++; 1775 mtfFreq[RUNB]++; 1776 } 1777 1778 if (zPend >= 2) { 1779 zPend = (zPend - 2) >> 1; 1780 } else { 1781 break; 1782 } 1783 } 1784 zPend = 0; 1785 } 1786 sfmap[wr] = (char) (j + 1); 1787 wr++; 1788 mtfFreq[j + 1]++; 1789 } 1790 } 1791 1792 if (zPend > 0) { 1793 zPend--; 1794 while (true) { 1795 if ((zPend & 1) == 0) { 1796 sfmap[wr] = RUNA; 1797 wr++; 1798 mtfFreq[RUNA]++; 1799 } else { 1800 sfmap[wr] = RUNB; 1801 wr++; 1802 mtfFreq[RUNB]++; 1803 } 1804 1805 if (zPend >= 2) { 1806 zPend = (zPend - 2) >> 1; 1807 } else { 1808 break; 1809 } 1810 } 1811 } 1812 1813 sfmap[wr] = (char) eob; 1814 mtfFreq[eob]++; 1815 this.nMTF = wr + 1; 1816 } 1817 1818 private static final class Data extends Object { 1819 1820 // with blockSize 900k 1821 final boolean[] inUse = new boolean[256]; // 256 byte 1822 final byte[] unseqToSeq = new byte[256]; // 256 byte 1823 final int[] mtfFreq = new int[MAX_ALPHA_SIZE]; // 1032 byte 1824 final byte[] selector = new byte[MAX_SELECTORS]; // 18002 byte 1825 final byte[] selectorMtf = new byte[MAX_SELECTORS]; // 18002 byte 1826 1827 final byte[] generateMTFValues_yy = new byte[256]; // 256 byte 1828 final byte[][] sendMTFValues_len = new byte[N_GROUPS][MAX_ALPHA_SIZE]; // 1548 1829 // byte 1830 final int[][] sendMTFValues_rfreq = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 1831 // byte 1832 final int[] sendMTFValues_fave = new int[N_GROUPS]; // 24 byte 1833 final short[] sendMTFValues_cost = new short[N_GROUPS]; // 12 byte 1834 final int[][] sendMTFValues_code = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 1835 // byte 1836 final byte[] sendMTFValues2_pos = new byte[N_GROUPS]; // 6 byte 1837 final boolean[] sentMTFValues4_inUse16 = new boolean[16]; // 16 byte 1838 1839 final int[] stack_ll = new int[QSORT_STACK_SIZE]; // 4000 byte 1840 final int[] stack_hh = new int[QSORT_STACK_SIZE]; // 4000 byte 1841 final int[] stack_dd = new int[QSORT_STACK_SIZE]; // 4000 byte 1842 1843 final int[] mainSort_runningOrder = new int[256]; // 1024 byte 1844 final int[] mainSort_copy = new int[256]; // 1024 byte 1845 final boolean[] mainSort_bigDone = new boolean[256]; // 256 byte 1846 1847 final int[] heap = new int[MAX_ALPHA_SIZE + 2]; // 1040 byte 1848 final int[] weight = new int[MAX_ALPHA_SIZE * 2]; // 2064 byte 1849 final int[] parent = new int[MAX_ALPHA_SIZE * 2]; // 2064 byte 1850 1851 final int[] ftab = new int[65537]; // 262148 byte 1852 // ------------ 1853 // 333408 byte 1854 1855 final byte[] block; // 900021 byte 1856 final int[] fmap; // 3600000 byte 1857 final char[] sfmap; // 3600000 byte 1858 // ------------ 1859 // 8433529 byte 1860 // ============ 1861 1862 /** 1863 * Array instance identical to sfmap, both are used only 1864 * temporarily and indepently, so we do not need to allocate 1865 * additional memory. 1866 */ 1867 final char[] quadrant; 1868 1869 Data(int blockSize100k) { 1870 super(); 1871 1872 final int n = blockSize100k * BZip2Constants.BASEBLOCKSIZE; 1873 this.block = new byte[(n + 1 + NUM_OVERSHOOT_BYTES)]; 1874 this.fmap = new int[n]; 1875 this.sfmap = new char[2 * n]; 1876 this.quadrant = this.sfmap; 1877 } 1878 1879 } 1880 1881 }