001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.net.ftp.parser;
019
020import java.text.ParseException;
021import java.util.List;
022
023import org.apache.commons.net.ftp.FTPClientConfig;
024import org.apache.commons.net.ftp.FTPFile;
025
026/**
027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
028 * Systems.
029 *
030 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
031 *      usage instructions)
032 */
033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
034
035    static final int UNKNOWN_LIST_TYPE = -1;
036    static final int FILE_LIST_TYPE = 0;
037    static final int MEMBER_LIST_TYPE = 1;
038    static final int UNIX_LIST_TYPE = 2;
039    static final int JES_LEVEL_1_LIST_TYPE = 3;
040    static final int JES_LEVEL_2_LIST_TYPE = 4;
041
042    private int isType = UNKNOWN_LIST_TYPE;
043
044    /**
045     * Fallback parser for Unix-style listings
046     */
047    private UnixFTPEntryParser unixFTPEntryParser;
048
049    /**
050     * Dates are ignored for file lists, but are used for member lists where
051     * possible
052     */
053    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
054                                                                    // 13:52
055
056    /**
057     * Matches these entries:
058     * <pre>
059     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
060     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
061     * </pre>
062     * @see https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm
063     */
064    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
065                                                                // ignored
066            "\\S+\\s+" + // unit - ignored
067            "\\S+\\s+" + // access date - ignored
068            "\\S+\\s+" + // extents -ignored
069            // If the values are too large, the fields may be merged (NET-639)
070            "(?:\\S+\\s+)?" + // used - ignored
071            "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U
072            "\\S+\\s+" + // logical record length -ignored
073            "\\S+\\s+" + // block size - ignored
074            "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
075            // but only support: PS, PO, PO-E
076            "(\\S+)\\s*"; // Dataset Name (file name)
077
078    /**
079     * Matches these entries:
080     * <pre>
081     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
082     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
083     * </pre>
084     */
085    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
086            "\\S+\\s+" + // version, modification (ignored)
087            "\\S+\\s+" + // create date (ignored)
088            "(\\S+)\\s+" + // modification date
089            "(\\S+)\\s+" + // modification time
090            "\\S+\\s+" + // size in lines (ignored)
091            "\\S+\\s+" + // size in lines at creation(ignored)
092            "\\S+\\s+" + // lines modified (ignored)
093            "\\S+\\s*"; // id of user who modified (ignored)
094
095    /**
096     * Matches these entries, note: no header:
097     * <pre>
098     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
099     *   012345678901234567890123456789012345678901234
100     *             1         2         3         4
101     * </pre>
102     */
103    static final String JES_LEVEL_1_LIST_REGEX =
104            "(\\S+)\\s+" + // job name ignored
105            "(\\S+)\\s+" + // job number
106            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
107            "(\\S+)\\s+" + // number of spool files
108            "(\\S+)\\s+" + // Text "Spool" ignored
109            "(\\S+)\\s*" // Text "Files" ignored
110    ;
111
112    /**
113     * JES INTERFACE LEVEL 2 parser
114     * Matches these entries:
115     * <pre>
116     * JOBNAME  JOBID    OWNER    STATUS CLASS
117     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
118     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
119     * </pre>
120     * Sample output from FTP session:
121     * <pre>
122     * ftp> quote site filetype=jes
123     * 200 SITE command was accepted
124     * ftp> ls
125     * 200 Port request OK.
126     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
127     * JOBNAME  JOBID    OWNER    STATUS CLASS
128     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
129     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
130     * 250 List completed successfully.
131     * ftp> ls job01906
132     * 200 Port request OK.
133     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
134     * JOBNAME  JOBID    OWNER    STATUS CLASS
135     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
136     * --------
137     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
138     * 001 JES2              A JESMSGLG       858
139     * 002 JES2              A JESJCL         128
140     * 003 JES2              A JESYSMSG       443
141     * 3 spool files
142     * 250 List completed successfully.
143     * </pre>
144     */
145
146    static final String JES_LEVEL_2_LIST_REGEX =
147            "(\\S+)\\s+" + // job name ignored
148            "(\\S+)\\s+" + // job number
149            "(\\S+)\\s+" + // owner ignored
150            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
151            "(\\S+)\\s+" + // job class ignored
152            "(\\S+).*" // rest ignored
153    ;
154
155    /*
156     * ---------------------------------------------------------------------
157     * Very brief and incomplete description of the zOS/MVS-file system. (Note:
158     * "zOS" is the operating system on the mainframe, and is the new name for
159     * MVS)
160     *
161     * The file system on the mainframe does not have hierarchal structure as for
162     * example the unix file system. For a more comprehensive description, please
163     * refer to the IBM manuals
164     *
165     * @LINK:
166     * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
167     *
168     *
169     * Dataset names =============
170     *
171     * A dataset name consist of a number of qualifiers separated by '.', each
172     * qualifier can be at most 8 characters, and the total length of a dataset
173     * can be max 44 characters including the dots.
174     *
175     *
176     * Dataset organisation ====================
177     *
178     * A dataset represents a piece of storage allocated on one or more disks.
179     * The structure of the storage is described with the field dataset
180     * organinsation (DSORG). There are a number of dataset organisations, but
181     * only two are usable for FTP transfer.
182     *
183     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
184     * extended partitioned dataset
185     *
186     * The PS file is just a flat file, as you would find it on the unix file
187     * system.
188     *
189     * The PO and PO-E files, can be compared to a single level directory
190     * structure. A PO file consist of a number of dataset members, or files if
191     * you will. It is possible to CD into the file, and to retrieve the
192     * individual members.
193     *
194     *
195     * Dataset record format =====================
196     *
197     * The physical layout of the dataset is described on the dataset itself.
198     * There are a number of record formats (RECFM), but just a few is relavant
199     * for the FTP transfer.
200     *
201     * Any one beginning with either F or V can safely used by FTP transfer. All
202     * others should only be used with great care.
203     * F means a fixed number of records per
204     * allocated storage, and V means a variable number of records.
205     *
206     *
207     * Other notes ===========
208     *
209     * The file system supports automatically backup and retrieval of datasets.
210     * If a file is backed up, the ftp LIST command will return: ARCIVE Not
211     * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
212     *
213     *
214     * Implementation notes ====================
215     *
216     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
217     * with F or V or U, is fully parsed.
218     *
219     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
220     * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
221     * FTPFile.Timestamp: change time or null
222     *
223     *
224     *
225     * Additional information ======================
226     *
227     * The MVS ftp server supports a number of features via the FTP interface.
228     * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
229     * SEQ is the default and used for normal file transfer JES is used to
230     * interact with the Job Entry Subsystem (JES) similar to a job scheduler
231     * DB2 is used to interact with a DB2 subsystem
232     *
233     * This parser supports SEQ and JES.
234     *
235     *
236     *
237     *
238     *
239     *
240     */
241
242    /**
243     * The sole constructor for a MVSFTPEntryParser object.
244     *
245     */
246    public MVSFTPEntryParser() {
247        super(""); // note the regex is set in preParse.
248        super.configure(null); // configure parser with default configurations
249    }
250
251    /**
252     * Parses a line of an z/OS - MVS FTP server file listing and converts it
253     * into a usable format in the form of an <code> FTPFile </code> instance.
254     * If the file listing line doesn't describe a file, then
255     * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
256     * instance representing the file is returned.
257     *
258     * @param entry
259     *            A line of text from the file listing
260     * @return An FTPFile instance corresponding to the supplied entry
261     */
262    @Override
263    public FTPFile parseFTPEntry(String entry) {
264        if (isType == FILE_LIST_TYPE) {
265            return parseFileList(entry);
266        } else if (isType == MEMBER_LIST_TYPE) {
267            return parseMemberList(entry);
268        } else if (isType == UNIX_LIST_TYPE) {
269             return unixFTPEntryParser.parseFTPEntry(entry);
270        } else if (isType == JES_LEVEL_1_LIST_TYPE) {
271            return parseJeslevel1List(entry);
272        } else if (isType == JES_LEVEL_2_LIST_TYPE) {
273            return parseJeslevel2List(entry);
274        }
275
276        return null;
277    }
278
279    /**
280     * Parse entries representing a dataset list. Only datasets with DSORG PS or
281     * PO or PO-E and with RECFM F[B], V[B], U will be parsed.
282     *
283     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
284     * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
285     * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
286     * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
287     * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
288     *
289     * ----------------------------------- Group within Regex [1] Volume [2]
290     * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
291     * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
292     * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
293     * Dsname: Dataset name
294     *
295     * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
296     * a tape archive. These entries is currently not supported by this parser.
297     * A null value is returned.
298     *
299     * @param entry zosDirectoryEntry
300     * @return null: entry was not parsed.
301     */
302    private FTPFile parseFileList(String entry) {
303        if (matches(entry)) {
304            FTPFile file = new FTPFile();
305            file.setRawListing(entry);
306            String name = group(2);
307            String dsorg = group(1);
308            file.setName(name);
309
310            // DSORG
311            if ("PS".equals(dsorg)) {
312                file.setType(FTPFile.FILE_TYPE);
313            }
314            else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
315                // regex already ruled out anything other than PO or PO-E
316                file.setType(FTPFile.DIRECTORY_TYPE);
317            }
318            else {
319                return null;
320            }
321
322            return file;
323        }
324
325        return null;
326    }
327
328    /**
329     * Parse entries within a partitioned dataset.
330     *
331     * Format of a memberlist within a PDS:
332     * <pre>
333     *    0         1        2          3        4     5     6      7    8
334     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
335     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
336     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
337     *
338     * -------------------------------------------
339     * [1] Name
340     * [2] VV.MM: Version . modification
341     * [3] Created: yyyy / MM / dd
342     * [4,5] Changed: yyyy / MM / dd HH:mm
343     * [6] Size: number of lines
344     * [7] Init: number of lines when first created
345     * [8] Mod: number of modified lines a last save
346     * [9] Id: User id for last update
347     * </pre>
348     *
349     * @param entry zosDirectoryEntry
350     * @return null: entry was not parsed.
351     */
352    private FTPFile parseMemberList(String entry) {
353        FTPFile file = new FTPFile();
354        if (matches(entry)) {
355            file.setRawListing(entry);
356            String name = group(1);
357            String datestr = group(2) + " " + group(3);
358            file.setName(name);
359            file.setType(FTPFile.FILE_TYPE);
360            try {
361                file.setTimestamp(super.parseTimestamp(datestr));
362            } catch (ParseException e) {
363                // just ignore parsing errors.
364                // TODO check this is ok
365                // Drop thru to try simple parser
366            }
367            return file;
368        }
369
370        /*
371         * Assigns the name to the first word of the entry. Only to be used from a
372         * safe context, for example from a memberlist, where the regex for some
373         * reason fails. Then just assign the name field of FTPFile.
374         */
375        if (entry != null && entry.trim().length() > 0) {
376            file.setRawListing(entry);
377            String name = entry.split(" ")[0];
378            file.setName(name);
379            file.setType(FTPFile.FILE_TYPE);
380            return file;
381        }
382        return null;
383    }
384
385    /**
386     * Matches these entries, note: no header:
387     * <pre>
388     * [1]      [2]      [3]   [4] [5]
389     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
390     * 012345678901234567890123456789012345678901234
391     *           1         2         3         4
392     * -------------------------------------------
393     * Group in regex
394     * [1] Job name
395     * [2] Job number
396     * [3] Job status (INPUT,ACTIVE,OUTPUT)
397     * [4] Number of sysout files
398     * [5] The string "Spool Files"
399     *</pre>
400     *
401     * @param entry zosDirectoryEntry
402     * @return null: entry was not parsed.
403     */
404    private FTPFile parseJeslevel1List(String entry) {
405        if (matches(entry)) {
406            FTPFile file = new FTPFile();
407            if (group(3).equalsIgnoreCase("OUTPUT")) {
408                file.setRawListing(entry);
409                String name = group(2); /* Job Number, used by GET */
410                file.setName(name);
411                file.setType(FTPFile.FILE_TYPE);
412                return file;
413            }
414        }
415
416        return null;
417    }
418
419    /**
420     * Matches these entries:
421     * <pre>
422     * [1]      [2]      [3]     [4]    [5]
423     * JOBNAME  JOBID    OWNER   STATUS CLASS
424     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
425     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
426     * 012345678901234567890123456789012345678901234
427     *           1         2         3         4
428     * -------------------------------------------
429     * Group in regex
430     * [1] Job name
431     * [2] Job number
432     * [3] Owner
433     * [4] Job status (INPUT,ACTIVE,OUTPUT)
434     * [5] Job Class
435     * [6] The rest
436     * </pre>
437     *
438     * @param entry zosDirectoryEntry
439     * @return null: entry was not parsed.
440     */
441    private FTPFile parseJeslevel2List(String entry) {
442        if (matches(entry)) {
443            FTPFile file = new FTPFile();
444            if (group(4).equalsIgnoreCase("OUTPUT")) {
445                file.setRawListing(entry);
446                String name = group(2); /* Job Number, used by GET */
447                file.setName(name);
448                file.setType(FTPFile.FILE_TYPE);
449                return file;
450            }
451        }
452
453        return null;
454    }
455
456    /**
457     * preParse is called as part of the interface. Per definition is is called
458     * before the parsing takes place.
459     * Three kind of lists is recognize:
460     * z/OS-MVS File lists
461     * z/OS-MVS Member lists
462     * unix file lists
463     * @since 2.0
464     */
465    @Override
466    public List<String> preParse(List<String> orig) {
467        // simply remove the header line. Composite logic will take care of the
468        // two different types of
469        // list in short order.
470        if (orig != null && orig.size() > 0) {
471            String header = orig.get(0);
472            if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
473                setType(FILE_LIST_TYPE);
474                super.setRegex(FILE_LIST_REGEX);
475            } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
476                setType(MEMBER_LIST_TYPE);
477                super.setRegex(MEMBER_LIST_REGEX);
478            } else if (header.indexOf("total") == 0) {
479                setType(UNIX_LIST_TYPE);
480                unixFTPEntryParser = new UnixFTPEntryParser();
481            } else if (header.indexOf("Spool Files") >= 30) {
482                setType(JES_LEVEL_1_LIST_TYPE);
483                super.setRegex(JES_LEVEL_1_LIST_REGEX);
484            } else if (header.indexOf("JOBNAME") == 0
485                    && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
486                setType(JES_LEVEL_2_LIST_TYPE);
487                super.setRegex(JES_LEVEL_2_LIST_REGEX);
488            } else {
489                setType(UNKNOWN_LIST_TYPE);
490            }
491
492            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
493                orig.remove(0);
494            }
495        }
496
497        return orig;
498    }
499
500    /**
501     * Explicitly set the type of listing being processed.
502     * @param type The listing type.
503     */
504    void setType(int type) {
505        isType = type;
506    }
507
508    /*
509     * @return
510     */
511    @Override
512    protected FTPClientConfig getDefaultConfiguration() {
513        return new FTPClientConfig(FTPClientConfig.SYST_MVS,
514                DEFAULT_DATE_FORMAT, null);
515    }
516
517}