View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.rng.examples.stress;
18  
19  import org.apache.commons.rng.UniformRandomProvider;
20  import org.apache.commons.rng.core.source64.RandomLongSource;
21  import org.apache.commons.rng.simple.RandomSource;
22  
23  import picocli.CommandLine.Command;
24  import picocli.CommandLine.Mixin;
25  import picocli.CommandLine.Option;
26  import picocli.CommandLine.Parameters;
27  
28  import java.io.BufferedWriter;
29  import java.io.File;
30  import java.io.FilterOutputStream;
31  import java.io.IOException;
32  import java.io.OutputStream;
33  import java.io.OutputStreamWriter;
34  import java.io.Writer;
35  import java.nio.ByteOrder;
36  import java.nio.charset.StandardCharsets;
37  import java.nio.file.Files;
38  import java.util.ArrayList;
39  import java.util.Formatter;
40  import java.util.List;
41  import java.util.concurrent.Callable;
42  
43  /**
44   * Specification for the "output" command.
45   *
46   * <p>This command creates a named random generator and outputs data in a specified format.</p>
47   */
48  @Command(name = "output",
49           description = {"Output data from a named random data generator."})
50  class OutputCommand implements Callable<Void> {
51      /** The new line characters. */
52      private static final String NEW_LINE = System.lineSeparator();
53      /** Character '['. */
54      private static final char LEFT_SQUARE_BRACKET = '[';
55      /** Character ']'. */
56      private static final char RIGHT_SQUARE_BRACKET = ']';
57  
58      /** Lookup table for binary representation of bytes. */
59      private static final String[] BIT_REP = {
60          "0000", "0001", "0010", "0011",
61          "0100", "0101", "0110", "0111",
62          "1000", "1001", "1010", "1011",
63          "1100", "1101", "1110", "1111",
64      };
65  
66      /** The standard options. */
67      @Mixin
68      private StandardOptions reusableOptions;
69  
70      /** The random source. */
71      @Parameters(index = "0",
72                  description = "The random source.")
73      private RandomSource randomSource;
74  
75      /** The executable arguments. */
76      @Parameters(index = "1..*",
77                  description = "The arguments to pass to the constructor.",
78                  paramLabel = "<argument>")
79      private List<String> arguments = new ArrayList<>();
80  
81      /** The file output prefix. */
82      @Option(names = {"-o", "--out"},
83              description = "The output file (default: stdout).")
84      private File fileOutput;
85  
86      /** The output format. */
87      @Option(names = {"-f", "--format"},
88              description = {"Output format (default: ${DEFAULT-VALUE}).",
89                             "Valid values: ${COMPLETION-CANDIDATES}."})
90      private OutputCommand.OutputFormat outputFormat = OutputFormat.DIEHARDER;
91  
92      /** The random seed. */
93      @Option(names = {"-s", "--seed"},
94              description = {"The 64-bit number random seed (default: auto)."})
95      private Long seed;
96  
97      /** The random seed as a byte[]. */
98      @Option(names = {"-x", "--hex-seed"},
99              description = {"The hex-encoded random seed.",
100                            "Seed conversion for multi-byte primitives use little-endian format.",
101                            "Over-rides the --seed parameter."})
102     private String byteSeed;
103 
104     /** The count of numbers to output. */
105     @Option(names = {"-n", "--count"},
106             description = {"The count of numbers to output.",
107                            "Use negative for an unlimited stream."})
108     private long count = 10;
109 
110     /** The size of the byte buffer for the binary data. */
111     @Option(names = {"--buffer-size"},
112             description = {"Byte-buffer size for binary data (default: ${DEFAULT-VALUE}).",
113                            "When outputing binary data the count parameter controls the " +
114                            "number of buffers written."})
115     private int bufferSize = 8192;
116 
117     /** The output byte order of the binary data. */
118     @Option(names = {"-b", "--byte-order"},
119             description = {"Byte-order of the output data (default: ${DEFAULT-VALUE}).",
120                            "Uses the Java default of big-endian. This may not match the platform byte-order.",
121                            "Valid values: BIG_ENDIAN, LITTLE_ENDIAN."})
122     private ByteOrder byteOrder = ByteOrder.BIG_ENDIAN;
123 
124     /** The output byte order of the binary data. */
125     @Option(names = {"-r", "--reverse-bits"},
126             description = {"Reverse the bits in the data (default: ${DEFAULT-VALUE})."})
127     private boolean reverseBits;
128 
129     /** Flag to use the upper 32-bits from the 64-bit long output. */
130     @Option(names = {"--high-bits"},
131             description = {"Use the upper 32-bits from the 64-bit long output.",
132                            "Takes precedent over --low-bits."})
133     private boolean longHighBits;
134 
135     /** Flag to use the lower 32-bits from the 64-bit long output. */
136     @Option(names = {"--low-bits"},
137             description = {"Use the lower 32-bits from the 64-bit long output."})
138     private boolean longLowBits;
139 
140     /** Flag to use 64-bit long output. */
141     @Option(names = {"--raw64"},
142             description = {"Use 64-bit output (default is 32-bit).",
143                            "This is ignored if not a native 64-bit generator.",
144                            "In 32-bit mode the output uses the upper then lower bits of 64-bit " +
145                            "generators sequentially."})
146     private boolean raw64;
147 
148     /**
149      * The output mode for existing files.
150      */
151     enum OutputFormat {
152         /** Binary output. */
153         BINARY,
154         /** Use the Dieharder text format. */
155         DIEHARDER,
156         /** Output the bits in a text format. */
157         BITS,
158     }
159 
160     /**
161      * Validates the command arguments, creates the generator and outputs numbers.
162      */
163     @Override
164     public Void call() {
165         LogUtils.setLogLevel(reusableOptions.logLevel);
166         final Object objectSeed = createSeed();
167         UniformRandomProvider rng = createRNG(objectSeed);
168 
169         // Upper or lower bits from 64-bit generators must be created first.
170         // This will throw if not a 64-bit generator.
171         if (longHighBits) {
172             rng = RNGUtils.createLongUpperBitsIntProvider(rng);
173         } else if (longLowBits) {
174             rng = RNGUtils.createLongLowerBitsIntProvider(rng);
175         }
176         if (reverseBits) {
177             rng = RNGUtils.createReverseBitsProvider(rng);
178         }
179 
180         // -------
181         // Note: Manipulation of the byte order for the platform is done during output
182         // for the binary format. Otherwise do it in Java.
183         // -------
184         if (outputFormat != OutputFormat.BINARY) {
185             rng = toOutputFormat(rng);
186         }
187 
188         try (OutputStream out = createOutputStream()) {
189             switch (outputFormat) {
190             case BINARY:
191                 writeBinaryData(rng, out);
192                 break;
193             case DIEHARDER:
194                 writeDieharder(rng, out);
195                 break;
196             case BITS:
197                 writeBitData(rng, out);
198                 break;
199             default:
200                 throw new ApplicationException("Unknown output format: " + outputFormat);
201             }
202         } catch (IOException ex) {
203             throw new ApplicationException("IO error: " + ex.getMessage(), ex);
204         }
205         return null;
206     }
207 
208     /**
209      * Creates the seed.
210      *
211      * @return the seed
212      */
213     private Object createSeed() {
214         if (byteSeed != null) {
215             try {
216                 return Hex.decodeHex(byteSeed);
217             } catch (IllegalArgumentException ex) {
218                 throw new ApplicationException("Invalid hex seed: " + ex.getMessage(), ex);
219             }
220         }
221         if (seed != null) {
222             return seed;
223         }
224         // Let the factory constructor create the native seed.
225         return null;
226     }
227 
228     /**
229      * Creates the seed.
230      *
231      * @return the seed
232      */
233     private String createSeedString() {
234         if (byteSeed != null) {
235             return byteSeed;
236         }
237         if (seed != null) {
238             return seed.toString();
239         }
240         return "auto";
241     }
242 
243     /**
244      * Creates the RNG.
245      *
246      * @param objectSeed Seed.
247      * @return the uniform random provider
248      * @throws ApplicationException If the RNG cannot be created
249      */
250     private UniformRandomProvider createRNG(Object objectSeed) {
251         if (randomSource == null) {
252             throw new ApplicationException("Random source is null");
253         }
254         final ArrayList<Object> data = new ArrayList<>();
255         // Note: The list command outputs arguments as an array bracketed by [ and ]
256         // Strip these for convenience.
257         stripArrayFormatting(arguments);
258 
259         for (final String argument : arguments) {
260             data.add(RNGUtils.parseArgument(argument));
261         }
262         try {
263             return RandomSource.create(randomSource, objectSeed, data.toArray());
264         } catch (IllegalStateException | IllegalArgumentException ex) {
265             throw new ApplicationException("Failed to create RNG: " + randomSource + ". " + ex.getMessage(), ex);
266         }
267     }
268 
269     /**
270      * Strip leading bracket from the first argument, trailing bracket from the last
271      * argument, and any trailing commas from any argument.
272      *
273      * <p>This is used to remove the array formatting used by the list command.
274      *
275      * @param arguments the arguments
276      */
277     private static void stripArrayFormatting(List<String> arguments) {
278         final int size = arguments.size();
279         if (size > 1) {
280             // These will not be empty as they were created from command-line args.
281             final String first = arguments.get(0);
282             if (first.charAt(0) == LEFT_SQUARE_BRACKET) {
283                 arguments.set(0, first.substring(1));
284             }
285             final String last = arguments.get(size - 1);
286             if (last.charAt(last.length() - 1) == RIGHT_SQUARE_BRACKET) {
287                 arguments.set(size - 1, last.substring(0, last.length() - 1));
288             }
289         }
290         for (int i = 0; i < size; i++) {
291             final String argument = arguments.get(i);
292             if (argument.endsWith(",")) {
293                 arguments.set(i, argument.substring(0, argument.length() - 1));
294             }
295         }
296     }
297 
298     /**
299      * Convert the native RNG to the requested output format. This will convert a 64-bit
300      * generator to a 32-bit generator unless the 64-bit mode is active. It then optionally
301      * reverses the byte order of the output.
302      *
303      * @param rng The random generator.
304      * @return the uniform random provider
305      */
306     private UniformRandomProvider toOutputFormat(UniformRandomProvider rng) {
307         UniformRandomProvider convertedRng = rng;
308         if (rng instanceof RandomLongSource && !raw64) {
309             // Convert to 32-bit generator
310             convertedRng = RNGUtils.createIntProvider(rng);
311         }
312         if (byteOrder == ByteOrder.LITTLE_ENDIAN) {
313             convertedRng = RNGUtils.createReverseBytesProvider(convertedRng);
314         }
315         return convertedRng;
316     }
317 
318     /**
319      * Creates the output stream. This will not be buffered.
320      *
321      * @return the output stream
322      */
323     private OutputStream createOutputStream() {
324         if (fileOutput != null) {
325             try {
326                 Files.newOutputStream(fileOutput.toPath());
327             } catch (IOException ex) {
328                 throw new ApplicationException("Failed to create output: " + fileOutput, ex);
329             }
330         }
331         return new FilterOutputStream(System.out) {
332             @Override
333             public void close() {
334                 // Do not close stdout
335             }
336         };
337     }
338 
339     /**
340      * Check the count is positive, otherwise create an error message for the provided format.
341      *
342      * @param count The count of numbers to output.
343      * @param format The format.
344      * @throws ApplicationException If the count is not positive.
345      */
346     private static void checkCount(long count,
347                                    OutputFormat format) {
348         if (count <= 0) {
349             throw new ApplicationException(format + " format requires a positive count: " + count);
350         }
351     }
352 
353     /**
354      * Write int data to the specified output using the dieharder text format.
355      *
356      * @param rng The random generator.
357      * @param out The output.
358      * @throws IOException Signals that an I/O exception has occurred.
359      * @throws ApplicationException If the count is not positive.
360      */
361     private void writeDieharder(final UniformRandomProvider rng,
362                                 final OutputStream out) throws IOException {
363         checkCount(count, OutputFormat.DIEHARDER);
364 
365         // Use dieharder output, e.g.
366         //#==================================================================
367         //# generator mt19937  seed = 1
368         //#==================================================================
369         //type: d
370         //count: 1
371         //numbit: 32
372         //1791095845
373         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
374             writeHeaderLine(output);
375             output.write("# generator ");
376             output.write(rng.toString());
377             output.write("  seed = ");
378             output.write(createSeedString());
379             output.write(NEW_LINE);
380             writeHeaderLine(output);
381             output.write("type: d");
382             output.write(NEW_LINE);
383             output.write("count: ");
384             output.write(Long.toString(count));
385             output.write(NEW_LINE);
386             output.write("numbit: 32");
387             output.write(NEW_LINE);
388             for (long c = 0; c < count; c++) {
389                 // Unsigned integers
390                 final String text = Long.toString(rng.nextInt() & 0xffffffffL);
391                 // Left pad with spaces
392                 for (int i = 10 - text.length(); i > 0; i--) {
393                     output.write(' ');
394                 }
395                 output.write(text);
396                 output.write(NEW_LINE);
397             }
398         }
399     }
400 
401     /**
402      * Write a header line to the output.
403      *
404      * @param output the output
405      * @throws IOException Signals that an I/O exception has occurred.
406      */
407     private static void writeHeaderLine(Writer output) throws IOException {
408         output.write("#==================================================================");
409         output.write(NEW_LINE);
410     }
411 
412     /**
413      * Write raw binary data to the output.
414      *
415      * @param rng The random generator.
416      * @param out The output.
417      * @throws IOException Signals that an I/O exception has occurred.
418      */
419     private void writeBinaryData(final UniformRandomProvider rng,
420                                  final OutputStream out) throws IOException {
421         // If count is not positive use max value.
422         // This is effectively unlimited: program must be killed.
423         final long limit = (count < 1) ? Long.MAX_VALUE : count;
424         try (RngDataOutput data = RNGUtils.createDataOutput(rng, raw64, out, bufferSize, byteOrder)) {
425             for (long c = 0; c < limit; c++) {
426                 data.write(rng);
427             }
428         }
429     }
430 
431     /**
432      * Write binary bit data to the specified file.
433      *
434      * @param rng The random generator.
435      * @param out The output.
436      * @throws IOException Signals that an I/O exception has occurred.
437      * @throws ApplicationException If the count is not positive.
438      */
439     private void writeBitData(final UniformRandomProvider rng,
440                               final OutputStream out) throws IOException {
441         checkCount(count, OutputFormat.BITS);
442 
443         boolean asLong = rng instanceof RandomLongSource;
444 
445         try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
446             for (long c = 0; c < count; c++) {
447                 if (asLong) {
448                     writeLong(output, rng.nextLong());
449                 } else {
450                     writeInt(output, rng.nextInt());
451                 }
452             }
453         }
454     }
455 
456     /**
457      * Write an {@code long} value to the the output. The native Java value will be
458      * written to the writer on a single line using: a binary string representation
459      * of the bytes; the unsigned integer; and the signed integer.
460      *
461      * <pre>
462      * 10011010 01010011 01011010 11100100 01000111 00010000 01000011 11000101  11120331841399178181 -7326412232310373435
463      * </pre>
464      *
465      * @param out The output.
466      * @param value The value.
467      * @throws IOException Signals that an I/O exception has occurred.
468      */
469     @SuppressWarnings("resource")
470     static void writeLong(Writer out,
471                           long value) throws IOException {
472 
473         // Write out as 8 bytes with spaces between them, high byte first.
474         writeByte(out, (int)(value >>> 56) & 0xff);
475         out.write(' ');
476         writeByte(out, (int)(value >>> 48) & 0xff);
477         out.write(' ');
478         writeByte(out, (int)(value >>> 40) & 0xff);
479         out.write(' ');
480         writeByte(out, (int)(value >>> 32) & 0xff);
481         out.write(' ');
482         writeByte(out, (int)(value >>> 24) & 0xff);
483         out.write(' ');
484         writeByte(out, (int)(value >>> 16) & 0xff);
485         out.write(' ');
486         writeByte(out, (int)(value >>>  8) & 0xff);
487         out.write(' ');
488         writeByte(out, (int)(value >>>  0) & 0xff);
489 
490         // Write the unsigned and signed int value
491         new Formatter(out).format("  %20s %20d%n", Long.toUnsignedString(value), value);
492     }
493 
494     /**
495      * Write an {@code int} value to the the output. The native Java value will be
496      * written to the writer on a single line using: a binary string representation
497      * of the bytes; the unsigned integer; and the signed integer.
498      *
499      * <pre>
500      * 11001101 00100011 01101111 01110000   3441651568  -853315728
501      * </pre>
502      *
503      * @param out The output.
504      * @param value The value.
505      * @throws IOException Signals that an I/O exception has occurred.
506      */
507     @SuppressWarnings("resource")
508     static void writeInt(Writer out,
509                          int value) throws IOException {
510 
511         // Write out as 4 bytes with spaces between them, high byte first.
512         writeByte(out, (value >>> 24) & 0xff);
513         out.write(' ');
514         writeByte(out, (value >>> 16) & 0xff);
515         out.write(' ');
516         writeByte(out, (value >>>  8) & 0xff);
517         out.write(' ');
518         writeByte(out, (value >>>  0) & 0xff);
519 
520         // Write the unsigned and signed int value
521         new Formatter(out).format("  %10d %11d%n", value & 0xffffffffL, value);
522     }
523 
524     /**
525      * Write the lower 8 bits of an {@code int} value to the buffered writer using a
526      * binary string representation. This is left-filled with zeros if applicable.
527      *
528      * <pre>
529      * 11001101
530      * </pre>
531      *
532      * @param out The output.
533      * @param value The value.
534      * @throws IOException Signals that an I/O exception has occurred.
535      */
536     private static void writeByte(Writer out,
537                                   int value) throws IOException {
538         // This matches the functionality of:
539         // data.write(String.format("%8s", Integer.toBinaryString(value & 0xff)).replace(' ', '0'))
540         out.write(BIT_REP[value >>> 4]);
541         out.write(BIT_REP[value & 0x0F]);
542     }
543 }