001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018 package org.apache.commons.net.ftp.parser;
019
020 import java.text.ParseException;
021 import java.util.List;
022
023 import org.apache.commons.net.ftp.FTPClientConfig;
024 import org.apache.commons.net.ftp.FTPFile;
025
026 /**
027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
028 * Systems.
029 *
030 * @author <a href="henrik.sorensen@balcab.ch">Henrik Sorensen</a>
031 * @author <a href="jnadler@srcginc.com">Jeff Nadler</a>
032 * @author <a href="wnoto@openfinance.com">William Noto</a>
033 *
034 * @version $Id: MVSFTPEntryParser.java 1325718 2012-04-13 11:13:51Z sebb $
035 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
036 * usage instructions)
037 */
038 public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
039
040 static final int UNKNOWN_LIST_TYPE = -1;
041 static final int FILE_LIST_TYPE = 0;
042 static final int MEMBER_LIST_TYPE = 1;
043 static final int UNIX_LIST_TYPE = 2;
044 static final int JES_LEVEL_1_LIST_TYPE = 3;
045 static final int JES_LEVEL_2_LIST_TYPE = 4;
046
047 private int isType = UNKNOWN_LIST_TYPE;
048
049 /**
050 * Fallback parser for Unix-style listings
051 */
052 private UnixFTPEntryParser unixFTPEntryParser;
053
054 /**
055 * Dates are ignored for file lists, but are used for member lists where
056 * possible
057 */
058 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
059 // 13:52
060
061 /**
062 * Matches these entries: Volume Unit Referred Ext Used Recfm Lrecl BlkSz
063 * Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
064 *
065 */
066 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
067 // ignored
068 "\\S+\\s+" + // unit - ignored
069 "\\S+\\s+" + // access date - ignored
070 "\\S+\\s+" + // extents -ignored
071 "\\S+\\s+" + // used - ignored
072 "[FV]\\S*\\s+" + // recfm - must start with F or V
073 "\\S+\\s+" + // logical record length -ignored
074 "\\S+\\s+" + // block size - ignored
075 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
076 // but only support: PS, PO, PO-E
077 "(\\S+)\\s*"; // Dataset Name (file name)
078
079 /**
080 * Matches these entries: Name VV.MM Created Changed Size Init Mod Id
081 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
082 */
083 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
084 "\\S+\\s+" + // version, modification (ignored)
085 "\\S+\\s+" + // create date (ignored)
086 "(\\S+)\\s+" + // modification date
087 "(\\S+)\\s+" + // modification time
088 "\\S+\\s+" + // size in lines (ignored)
089 "\\S+\\s+" + // size in lines at creation(ignored)
090 "\\S+\\s+" + // lines modified (ignored)
091 "\\S+\\s*"; // id of user who modified (ignored)
092
093 /**
094 * Matches these entries, note: no header: IBMUSER1 JOB01906 OUTPUT 3 Spool
095 * Files 012345678901234567890123456789012345678901234 1 2 3 4
096 */
097 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job
098 // name
099 // ignored
100 "(\\S+)\\s+" + // job number
101 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
102 "(\\S+)\\s+" + // number of spool files
103 "(\\S+)\\s+" + // Text "Spool" ignored
104 "(\\S+)\\s*" // Text "Files" ignored
105 ;
106
107 /**
108 * JES INTERFACE LEVEL 2 parser Matches these entries: JOBNAME JOBID OWNER
109 * STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
110 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
111 * 012345678901234567890123456789012345678901234 1 2 3 4
112 * 012345678901234567890123456789012345678901234567890
113 */
114
115 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job
116 // name
117 // ignored
118 "(\\S+)\\s+" + // job number
119 "(\\S+)\\s+" + // owner ignored
120 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
121 "(\\S+)\\s+" + // job class ignored
122 "(\\S+).*" // rest ignored
123 ;
124
125 /*
126 * ---------------------------------------------------------------------
127 * Very brief and incomplete description of the zOS/MVS-filesystem. (Note:
128 * "zOS" is the operating system on the mainframe, and is the new name for
129 * MVS)
130 *
131 * The filesystem on the mainframe does not have hierarchal structure as for
132 * example the unix filesystem. For a more comprehensive description, please
133 * refer to the IBM manuals
134 *
135 * @LINK:
136 * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
137 *
138 *
139 * Dataset names =============
140 *
141 * A dataset name consist of a number of qualifiers separated by '.', each
142 * qualifier can be at most 8 characters, and the total length of a dataset
143 * can be max 44 characters including the dots.
144 *
145 *
146 * Dataset organisation ====================
147 *
148 * A dataset represents a piece of storage allocated on one or more disks.
149 * The structure of the storage is described with the field dataset
150 * organinsation (DSORG). There are a number of dataset organisations, but
151 * only two are usable for FTP transfer.
152 *
153 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
154 * extended partitioned dataset
155 *
156 * The PS file is just a flat file, as you would find it on the unix file
157 * system.
158 *
159 * The PO and PO-E files, can be compared to a single level directory
160 * structure. A PO file consist of a number of dataset members, or files if
161 * you will. It is possible to CD into the file, and to retrieve the
162 * individual members.
163 *
164 *
165 * Dataset record format =====================
166 *
167 * The physical layout of the dataset is described on the dataset itself.
168 * There are a number of record formats (RECFM), but just a few is relavant
169 * for the FTP transfer.
170 *
171 * Any one beginning with either F or V can safely used by FTP transfer. All
172 * others should only be used with great care, so this version will just
173 * ignore the other record formats. F means a fixed number of records per
174 * allocated storage, and V means a variable number of records.
175 *
176 *
177 * Other notes ===========
178 *
179 * The file system supports automatically backup and retrieval of datasets.
180 * If a file is backed up, the ftp LIST command will return: ARCIVE Not
181 * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
182 *
183 *
184 * Implementation notes ====================
185 *
186 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
187 * with F or V, is fully parsed.
188 *
189 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
190 * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
191 * FTPFile.Timestamp: change time or null
192 *
193 *
194 *
195 * Additional information ======================
196 *
197 * The MVS ftp server supports a number of features via the FTP interface.
198 * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
199 * SEQ is the default and used for normal file transfer JES is used to
200 * interact with the Job Entry Subsystem (JES) similar to a job scheduler
201 * DB2 is used to interact with a DB2 subsystem
202 *
203 * This parser supports SEQ and JES.
204 *
205 *
206 *
207 *
208 *
209 *
210 */
211
212 /**
213 * The sole constructor for a MVSFTPEntryParser object.
214 *
215 */
216 public MVSFTPEntryParser() {
217 super(""); // note the regex is set in preParse.
218 super.configure(null); // configure parser with default configurations
219 }
220
221 /**
222 * Parses a line of an z/OS - MVS FTP server file listing and converts it
223 * into a usable format in the form of an <code> FTPFile </code> instance.
224 * If the file listing line doesn't describe a file, then
225 * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
226 * instance representing the file is returned.
227 *
228 * @param entry
229 * A line of text from the file listing
230 * @return An FTPFile instance corresponding to the supplied entry
231 */
232 public FTPFile parseFTPEntry(String entry) {
233 boolean isParsed = false;
234 FTPFile f = new FTPFile();
235
236 if (isType == FILE_LIST_TYPE) {
237 isParsed = parseFileList(f, entry);
238 } else if (isType == MEMBER_LIST_TYPE) {
239 isParsed = parseMemberList(f, entry);
240 if (!isParsed) {
241 isParsed = parseSimpleEntry(f, entry);
242 }
243 } else if (isType == UNIX_LIST_TYPE) {
244 isParsed = parseUnixList(f, entry);
245 } else if (isType == JES_LEVEL_1_LIST_TYPE) {
246 isParsed = parseJeslevel1List(f, entry);
247 } else if (isType == JES_LEVEL_2_LIST_TYPE) {
248 isParsed = parseJeslevel2List(f, entry);
249 }
250
251 if (!isParsed) {
252 f = null;
253 }
254
255 return f;
256 }
257
258 /**
259 * Parse entries representing a dataset list. Only datasets with DSORG PS or
260 * PO or PO-E and with RECFM F* or V* will be parsed.
261 *
262 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
263 * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
264 * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
265 * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
266 * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
267 *
268 * ----------------------------------- Group within Regex [1] Volume [2]
269 * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
270 * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
271 * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
272 * Dsname: Dataset name
273 *
274 * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
275 * a tape archive. These entries is currently not supported by this parser.
276 * A null value is returned.
277 *
278 * @param file
279 * will be updated with Name, Type, Timestamp if parsed.
280 * @param entry zosDirectoryEntry
281 * @return true: entry was parsed, false: entry was not parsed.
282 */
283 private boolean parseFileList(FTPFile file, String entry) {
284 if (matches(entry)) {
285 file.setRawListing(entry);
286 String name = group(2);
287 String dsorg = group(1);
288 file.setName(name);
289
290 // DSORG
291 if ("PS".equals(dsorg)) {
292 file.setType(FTPFile.FILE_TYPE);
293 }
294 else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
295 // regex already ruled out anything other than PO or PO-E
296 file.setType(FTPFile.DIRECTORY_TYPE);
297 }
298 else {
299 return false;
300 }
301
302 return true;
303 }
304
305 return false;
306 }
307
308 /**
309 * Parse entries within a partitioned dataset.
310 *
311 * Format of a memberlist within a PDS: 1 2 3 4 5 6 7 8 9 Name VV.MM Created
312 * Changed Size Init Mod Id TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11
313 * 0 KIL001 TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001
314 *
315 * ------------------------------------------- [1] Name [2] VV.MM: Version .
316 * modification [3] Created: yyyy / MM / dd [4,5] Changed: yyyy / MM / dd
317 * HH:mm [6] Size: number of lines [7] Init: number of lines when first
318 * created [8] Mod: number of modified lines a last save [9] Id: User id for
319 * last update
320 *
321 *
322 * @param file
323 * will be updated with Name, Type and Timestamp if parsed.
324 * @param entry zosDirectoryEntry
325 * @return true: entry was parsed, false: entry was not parsed.
326 */
327 private boolean parseMemberList(FTPFile file, String entry) {
328 if (matches(entry)) {
329 file.setRawListing(entry);
330 String name = group(1);
331 String datestr = group(2) + " " + group(3);
332 file.setName(name);
333 file.setType(FTPFile.FILE_TYPE);
334 try {
335 file.setTimestamp(super.parseTimestamp(datestr));
336 } catch (ParseException e) {
337 e.printStackTrace();
338 // just ignore parsing errors.
339 // TODO check this is ok
340 return false; // this is a parsing failure too.
341 }
342 return true;
343 }
344
345 return false;
346 }
347
348 /**
349 * Assigns the name to the first word of the entry. Only to be used from a
350 * safe context, for example from a memberlist, where the regex for some
351 * reason fails. Then just assign the name field of FTPFile.
352 *
353 * @param file
354 * @param entry
355 * @return
356 */
357 private boolean parseSimpleEntry(FTPFile file, String entry) {
358 if (entry != null && entry.trim().length() > 0) {
359 file.setRawListing(entry);
360 String name = entry.split(" ")[0];
361 file.setName(name);
362 file.setType(FTPFile.FILE_TYPE);
363 return true;
364 }
365 return false;
366 }
367
368 /**
369 * Parse the entry as a standard unix file. Using the UnixFTPEntryParser.
370 *
371 * @param file
372 * @param entry
373 * @return true: entry is parsed, false: entry could not be parsed.
374 */
375 private boolean parseUnixList(FTPFile file, String entry) {
376 file = unixFTPEntryParser.parseFTPEntry(entry);
377 if (file == null) {
378 return false;
379 }
380 return true;
381 }
382
383 /**
384 * Matches these entries, note: no header: [1] [2] [3] [4] [5] IBMUSER1
385 * JOB01906 OUTPUT 3 Spool Files
386 * 012345678901234567890123456789012345678901234 1 2 3 4
387 * ------------------------------------------- Group in regex [1] Job name
388 * [2] Job number [3] Job status (INPUT,ACTIVE,OUTPUT) [4] Number of sysout
389 * files [5] The string "Spool Files"
390 *
391 *
392 * @param file
393 * will be updated with Name, Type and Timestamp if parsed.
394 * @param entry zosDirectoryEntry
395 * @return true: entry was parsed, false: entry was not parsed.
396 */
397 private boolean parseJeslevel1List(FTPFile file, String entry) {
398 if (matches(entry)) {
399 if (group(3).equalsIgnoreCase("OUTPUT")) {
400 file.setRawListing(entry);
401 String name = group(2); /* Job Number, used by GET */
402 file.setName(name);
403 file.setType(FTPFile.FILE_TYPE);
404 return true;
405 }
406 }
407
408 return false;
409 }
410
411 /**
412 * Matches these entries, note: no header: [1] [2] [3] [4] [5] JOBNAME JOBID
413 * OWNER STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool
414 * files IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
415 * 012345678901234567890123456789012345678901234 1 2 3 4
416 * ------------------------------------------- Group in regex [1] Job name
417 * [2] Job number [3] Owner [4] Job status (INPUT,ACTIVE,OUTPUT) [5] Job
418 * Class [6] The rest
419 *
420 *
421 * @param file
422 * will be updated with Name, Type and Timestamp if parsed.
423 * @param entry zosDirectoryEntry
424 * @return true: entry was parsed, false: entry was not parsed.
425 */
426 private boolean parseJeslevel2List(FTPFile file, String entry) {
427 if (matches(entry)) {
428 if (group(4).equalsIgnoreCase("OUTPUT")) {
429 file.setRawListing(entry);
430 String name = group(2); /* Job Number, used by GET */
431 file.setName(name);
432 file.setType(FTPFile.FILE_TYPE);
433 return true;
434 }
435 }
436
437 return false;
438 }
439
440 /**
441 * preParse is called as part of the interface. Per definition is is called
442 * before the parsing takes place. Three kind of lists is recognize:
443 * z/OS-MVS File lists z/OS-MVS Member lists unix file lists
444 * @since 2.0
445 */
446 @Override
447 public List<String> preParse(List<String> orig) {
448 // simply remove the header line. Composite logic will take care of the
449 // two different types of
450 // list in short order.
451 if (orig != null && orig.size() > 0) {
452 String header = orig.get(0);
453 if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
454 setType(FILE_LIST_TYPE);
455 super.setRegex(FILE_LIST_REGEX);
456 } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
457 setType(MEMBER_LIST_TYPE);
458 super.setRegex(MEMBER_LIST_REGEX);
459 } else if (header.indexOf("total") == 0) {
460 setType(UNIX_LIST_TYPE);
461 unixFTPEntryParser = new UnixFTPEntryParser();
462 } else if (header.indexOf("Spool Files") >= 30) {
463 setType(JES_LEVEL_1_LIST_TYPE);
464 super.setRegex(JES_LEVEL_1_LIST_REGEX);
465 } else if (header.indexOf("JOBNAME") == 0
466 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
467 setType(JES_LEVEL_2_LIST_TYPE);
468 super.setRegex(JES_LEVEL_2_LIST_REGEX);
469 } else {
470 setType(UNKNOWN_LIST_TYPE);
471 }
472
473 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
474 orig.remove(0);
475 }
476 }
477
478 return orig;
479 }
480
481 /**
482 * Explicitly set the type of listing being processed.
483 * @param type The listing type.
484 */
485 void setType(int type) {
486 isType = type;
487 }
488
489 /*
490 * @return
491 */
492 @Override
493 protected FTPClientConfig getDefaultConfiguration() {
494 return new FTPClientConfig(FTPClientConfig.SYST_MVS,
495 DEFAULT_DATE_FORMAT, null, null, null, null);
496 }
497
498 }