001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.FTPClientConfig; 024import org.apache.commons.net.ftp.FTPFile; 025 026/** 027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS 028 * Systems. 029 * 030 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for 031 * usage instructions) 032 */ 033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 034 035 static final int UNKNOWN_LIST_TYPE = -1; 036 static final int FILE_LIST_TYPE = 0; 037 static final int MEMBER_LIST_TYPE = 1; 038 static final int UNIX_LIST_TYPE = 2; 039 static final int JES_LEVEL_1_LIST_TYPE = 3; 040 static final int JES_LEVEL_2_LIST_TYPE = 4; 041 042 private int isType = UNKNOWN_LIST_TYPE; 043 044 /** 045 * Fallback parser for Unix-style listings 046 */ 047 private UnixFTPEntryParser unixFTPEntryParser; 048 049 /** 050 * Dates are ignored for file lists, but are used for member lists where 051 * possible 052 */ 053 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 054 // 13:52 055 056 /** 057 * Matches these entries: 058 * <pre> 059 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 060 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 061 * </pre> 062 * @see https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm 063 */ 064 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 065 // ignored 066 "\\S+\\s+" + // unit - ignored 067 "\\S+\\s+" + // access date - ignored 068 "\\S+\\s+" + // extents -ignored 069 // If the values are too large, the fields may be merged (NET-639) 070 "(?:\\S+\\s+)?" + // used - ignored 071 "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U 072 "\\S+\\s+" + // logical record length -ignored 073 "\\S+\\s+" + // block size - ignored 074 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 075 // but only support: PS, PO, PO-E 076 "(\\S+)\\s*"; // Dataset Name (file name) 077 078 /** 079 * Matches these entries: 080 * <pre> 081 * Name VV.MM Created Changed Size Init Mod Id 082 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 083 * </pre> 084 */ 085 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 086 "\\S+\\s+" + // version, modification (ignored) 087 "\\S+\\s+" + // create date (ignored) 088 "(\\S+)\\s+" + // modification date 089 "(\\S+)\\s+" + // modification time 090 "\\S+\\s+" + // size in lines (ignored) 091 "\\S+\\s+" + // size in lines at creation(ignored) 092 "\\S+\\s+" + // lines modified (ignored) 093 "\\S+\\s*"; // id of user who modified (ignored) 094 095 /** 096 * Matches these entries, note: no header: 097 * <pre> 098 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 099 * 012345678901234567890123456789012345678901234 100 * 1 2 3 4 101 * </pre> 102 */ 103 static final String JES_LEVEL_1_LIST_REGEX = 104 "(\\S+)\\s+" + // job name ignored 105 "(\\S+)\\s+" + // job number 106 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 107 "(\\S+)\\s+" + // number of spool files 108 "(\\S+)\\s+" + // Text "Spool" ignored 109 "(\\S+)\\s*" // Text "Files" ignored 110 ; 111 112 /** 113 * JES INTERFACE LEVEL 2 parser 114 * Matches these entries: 115 * <pre> 116 * JOBNAME JOBID OWNER STATUS CLASS 117 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 118 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 119 * </pre> 120 * Sample output from FTP session: 121 * <pre> 122 * ftp> quote site filetype=jes 123 * 200 SITE command was accepted 124 * ftp> ls 125 * 200 Port request OK. 126 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 127 * JOBNAME JOBID OWNER STATUS CLASS 128 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 129 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 130 * 250 List completed successfully. 131 * ftp> ls job01906 132 * 200 Port request OK. 133 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 134 * JOBNAME JOBID OWNER STATUS CLASS 135 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 136 * -------- 137 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 138 * 001 JES2 A JESMSGLG 858 139 * 002 JES2 A JESJCL 128 140 * 003 JES2 A JESYSMSG 443 141 * 3 spool files 142 * 250 List completed successfully. 143 * </pre> 144 */ 145 146 static final String JES_LEVEL_2_LIST_REGEX = 147 "(\\S+)\\s+" + // job name ignored 148 "(\\S+)\\s+" + // job number 149 "(\\S+)\\s+" + // owner ignored 150 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 151 "(\\S+)\\s+" + // job class ignored 152 "(\\S+).*" // rest ignored 153 ; 154 155 /* 156 * --------------------------------------------------------------------- 157 * Very brief and incomplete description of the zOS/MVS-file system. (Note: 158 * "zOS" is the operating system on the mainframe, and is the new name for 159 * MVS) 160 * 161 * The file system on the mainframe does not have hierarchal structure as for 162 * example the unix file system. For a more comprehensive description, please 163 * refer to the IBM manuals 164 * 165 * @LINK: 166 * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 167 * 168 * 169 * Dataset names ============= 170 * 171 * A dataset name consist of a number of qualifiers separated by '.', each 172 * qualifier can be at most 8 characters, and the total length of a dataset 173 * can be max 44 characters including the dots. 174 * 175 * 176 * Dataset organisation ==================== 177 * 178 * A dataset represents a piece of storage allocated on one or more disks. 179 * The structure of the storage is described with the field dataset 180 * organinsation (DSORG). There are a number of dataset organisations, but 181 * only two are usable for FTP transfer. 182 * 183 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: 184 * extended partitioned dataset 185 * 186 * The PS file is just a flat file, as you would find it on the unix file 187 * system. 188 * 189 * The PO and PO-E files, can be compared to a single level directory 190 * structure. A PO file consist of a number of dataset members, or files if 191 * you will. It is possible to CD into the file, and to retrieve the 192 * individual members. 193 * 194 * 195 * Dataset record format ===================== 196 * 197 * The physical layout of the dataset is described on the dataset itself. 198 * There are a number of record formats (RECFM), but just a few is relavant 199 * for the FTP transfer. 200 * 201 * Any one beginning with either F or V can safely used by FTP transfer. All 202 * others should only be used with great care. 203 * F means a fixed number of records per 204 * allocated storage, and V means a variable number of records. 205 * 206 * 207 * Other notes =========== 208 * 209 * The file system supports automatically backup and retrieval of datasets. 210 * If a file is backed up, the ftp LIST command will return: ARCIVE Not 211 * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 212 * 213 * 214 * Implementation notes ==================== 215 * 216 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning 217 * with F or V or U, is fully parsed. 218 * 219 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. 220 * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 221 * FTPFile.Timestamp: change time or null 222 * 223 * 224 * 225 * Additional information ====================== 226 * 227 * The MVS ftp server supports a number of features via the FTP interface. 228 * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2> 229 * SEQ is the default and used for normal file transfer JES is used to 230 * interact with the Job Entry Subsystem (JES) similar to a job scheduler 231 * DB2 is used to interact with a DB2 subsystem 232 * 233 * This parser supports SEQ and JES. 234 * 235 * 236 * 237 * 238 * 239 * 240 */ 241 242 /** 243 * The sole constructor for a MVSFTPEntryParser object. 244 * 245 */ 246 public MVSFTPEntryParser() { 247 super(""); // note the regex is set in preParse. 248 super.configure(null); // configure parser with default configurations 249 } 250 251 /** 252 * Parses a line of an z/OS - MVS FTP server file listing and converts it 253 * into a usable format in the form of an <code> FTPFile </code> instance. 254 * If the file listing line doesn't describe a file, then 255 * <code> null </code> is returned. Otherwise a <code> FTPFile </code> 256 * instance representing the file is returned. 257 * 258 * @param entry 259 * A line of text from the file listing 260 * @return An FTPFile instance corresponding to the supplied entry 261 */ 262 @Override 263 public FTPFile parseFTPEntry(String entry) { 264 if (isType == FILE_LIST_TYPE) { 265 return parseFileList(entry); 266 } else if (isType == MEMBER_LIST_TYPE) { 267 return parseMemberList(entry); 268 } else if (isType == UNIX_LIST_TYPE) { 269 return unixFTPEntryParser.parseFTPEntry(entry); 270 } else if (isType == JES_LEVEL_1_LIST_TYPE) { 271 return parseJeslevel1List(entry); 272 } else if (isType == JES_LEVEL_2_LIST_TYPE) { 273 return parseJeslevel2List(entry); 274 } 275 276 return null; 277 } 278 279 /** 280 * Parse entries representing a dataset list. Only datasets with DSORG PS or 281 * PO or PO-E and with RECFM F[B], V[B], U will be parsed. 282 * 283 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred 284 * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 285 * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device 286 * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO 287 * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 288 * 289 * ----------------------------------- Group within Regex [1] Volume [2] 290 * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record 291 * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: 292 * Dataset organisation. Many exists but only support: PS, PO, PO-E [10] 293 * Dsname: Dataset name 294 * 295 * Note: When volume is ARCIVE, it means the dataset is stored somewhere in 296 * a tape archive. These entries is currently not supported by this parser. 297 * A null value is returned. 298 * 299 * @param entry zosDirectoryEntry 300 * @return null: entry was not parsed. 301 */ 302 private FTPFile parseFileList(String entry) { 303 if (matches(entry)) { 304 FTPFile file = new FTPFile(); 305 file.setRawListing(entry); 306 String name = group(2); 307 String dsorg = group(1); 308 file.setName(name); 309 310 // DSORG 311 if ("PS".equals(dsorg)) { 312 file.setType(FTPFile.FILE_TYPE); 313 } 314 else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 315 // regex already ruled out anything other than PO or PO-E 316 file.setType(FTPFile.DIRECTORY_TYPE); 317 } 318 else { 319 return null; 320 } 321 322 return file; 323 } 324 325 return null; 326 } 327 328 /** 329 * Parse entries within a partitioned dataset. 330 * 331 * Format of a memberlist within a PDS: 332 * <pre> 333 * 0 1 2 3 4 5 6 7 8 334 * Name VV.MM Created Changed Size Init Mod Id 335 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 336 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 337 * 338 * ------------------------------------------- 339 * [1] Name 340 * [2] VV.MM: Version . modification 341 * [3] Created: yyyy / MM / dd 342 * [4,5] Changed: yyyy / MM / dd HH:mm 343 * [6] Size: number of lines 344 * [7] Init: number of lines when first created 345 * [8] Mod: number of modified lines a last save 346 * [9] Id: User id for last update 347 * </pre> 348 * 349 * @param entry zosDirectoryEntry 350 * @return null: entry was not parsed. 351 */ 352 private FTPFile parseMemberList(String entry) { 353 FTPFile file = new FTPFile(); 354 if (matches(entry)) { 355 file.setRawListing(entry); 356 String name = group(1); 357 String datestr = group(2) + " " + group(3); 358 file.setName(name); 359 file.setType(FTPFile.FILE_TYPE); 360 try { 361 file.setTimestamp(super.parseTimestamp(datestr)); 362 } catch (ParseException e) { 363 // just ignore parsing errors. 364 // TODO check this is ok 365 // Drop thru to try simple parser 366 } 367 return file; 368 } 369 370 /* 371 * Assigns the name to the first word of the entry. Only to be used from a 372 * safe context, for example from a memberlist, where the regex for some 373 * reason fails. Then just assign the name field of FTPFile. 374 */ 375 if (entry != null && entry.trim().length() > 0) { 376 file.setRawListing(entry); 377 String name = entry.split(" ")[0]; 378 file.setName(name); 379 file.setType(FTPFile.FILE_TYPE); 380 return file; 381 } 382 return null; 383 } 384 385 /** 386 * Matches these entries, note: no header: 387 * <pre> 388 * [1] [2] [3] [4] [5] 389 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 390 * 012345678901234567890123456789012345678901234 391 * 1 2 3 4 392 * ------------------------------------------- 393 * Group in regex 394 * [1] Job name 395 * [2] Job number 396 * [3] Job status (INPUT,ACTIVE,OUTPUT) 397 * [4] Number of sysout files 398 * [5] The string "Spool Files" 399 *</pre> 400 * 401 * @param entry zosDirectoryEntry 402 * @return null: entry was not parsed. 403 */ 404 private FTPFile parseJeslevel1List(String entry) { 405 if (matches(entry)) { 406 FTPFile file = new FTPFile(); 407 if (group(3).equalsIgnoreCase("OUTPUT")) { 408 file.setRawListing(entry); 409 String name = group(2); /* Job Number, used by GET */ 410 file.setName(name); 411 file.setType(FTPFile.FILE_TYPE); 412 return file; 413 } 414 } 415 416 return null; 417 } 418 419 /** 420 * Matches these entries: 421 * <pre> 422 * [1] [2] [3] [4] [5] 423 * JOBNAME JOBID OWNER STATUS CLASS 424 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 425 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 426 * 012345678901234567890123456789012345678901234 427 * 1 2 3 4 428 * ------------------------------------------- 429 * Group in regex 430 * [1] Job name 431 * [2] Job number 432 * [3] Owner 433 * [4] Job status (INPUT,ACTIVE,OUTPUT) 434 * [5] Job Class 435 * [6] The rest 436 * </pre> 437 * 438 * @param entry zosDirectoryEntry 439 * @return null: entry was not parsed. 440 */ 441 private FTPFile parseJeslevel2List(String entry) { 442 if (matches(entry)) { 443 FTPFile file = new FTPFile(); 444 if (group(4).equalsIgnoreCase("OUTPUT")) { 445 file.setRawListing(entry); 446 String name = group(2); /* Job Number, used by GET */ 447 file.setName(name); 448 file.setType(FTPFile.FILE_TYPE); 449 return file; 450 } 451 } 452 453 return null; 454 } 455 456 /** 457 * preParse is called as part of the interface. Per definition is is called 458 * before the parsing takes place. 459 * Three kind of lists is recognize: 460 * z/OS-MVS File lists 461 * z/OS-MVS Member lists 462 * unix file lists 463 * @since 2.0 464 */ 465 @Override 466 public List<String> preParse(List<String> orig) { 467 // simply remove the header line. Composite logic will take care of the 468 // two different types of 469 // list in short order. 470 if (orig != null && orig.size() > 0) { 471 String header = orig.get(0); 472 if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) { 473 setType(FILE_LIST_TYPE); 474 super.setRegex(FILE_LIST_REGEX); 475 } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) { 476 setType(MEMBER_LIST_TYPE); 477 super.setRegex(MEMBER_LIST_REGEX); 478 } else if (header.indexOf("total") == 0) { 479 setType(UNIX_LIST_TYPE); 480 unixFTPEntryParser = new UnixFTPEntryParser(); 481 } else if (header.indexOf("Spool Files") >= 30) { 482 setType(JES_LEVEL_1_LIST_TYPE); 483 super.setRegex(JES_LEVEL_1_LIST_REGEX); 484 } else if (header.indexOf("JOBNAME") == 0 485 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS 486 setType(JES_LEVEL_2_LIST_TYPE); 487 super.setRegex(JES_LEVEL_2_LIST_REGEX); 488 } else { 489 setType(UNKNOWN_LIST_TYPE); 490 } 491 492 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 493 orig.remove(0); 494 } 495 } 496 497 return orig; 498 } 499 500 /** 501 * Explicitly set the type of listing being processed. 502 * @param type The listing type. 503 */ 504 void setType(int type) { 505 isType = type; 506 } 507 508 /* 509 * @return 510 */ 511 @Override 512 protected FTPClientConfig getDefaultConfiguration() { 513 return new FTPClientConfig(FTPClientConfig.SYST_MVS, 514 DEFAULT_DATE_FORMAT, null); 515 } 516 517}