001 /*
002 * $Id: AtcfStormDataSource.java,v 1.1 2012/01/04 20:40:51 tommyj Exp $
003 *
004 * This file is part of McIDAS-V
005 *
006 * Copyright 2007-2012
007 * Space Science and Engineering Center (SSEC)
008 * University of Wisconsin - Madison
009 * 1225 W. Dayton Street, Madison, WI 53706, USA
010 * https://www.ssec.wisc.edu/mcidas
011 *
012 * All Rights Reserved
013 *
014 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
015 * some McIDAS-V source code is based on IDV and VisAD source code.
016 *
017 * McIDAS-V is free software; you can redistribute it and/or modify
018 * it under the terms of the GNU Lesser Public License as published by
019 * the Free Software Foundation; either version 3 of the License, or
020 * (at your option) any later version.
021 *
022 * McIDAS-V is distributed in the hope that it will be useful,
023 * but WITHOUT ANY WARRANTY; without even the implied warranty of
024 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
025 * GNU Lesser Public License for more details.
026 *
027 * You should have received a copy of the GNU Lesser Public License
028 * along with this program. If not, see http://www.gnu.org/licenses.
029 */
030
031 package edu.wisc.ssec.mcidasv.data.cyclone;
032
033 import java.io.ByteArrayInputStream;
034 import java.io.ByteArrayOutputStream;
035 import java.io.File;
036 import java.io.FileNotFoundException;
037 import java.io.IOException;
038 import java.net.URL;
039 import java.text.SimpleDateFormat;
040 import java.util.ArrayList;
041 import java.util.Calendar;
042 import java.util.Date;
043 import java.util.GregorianCalendar;
044 import java.util.Hashtable;
045 import java.util.List;
046 import java.util.TimeZone;
047 import java.util.zip.GZIPInputStream;
048
049 import org.apache.commons.net.ftp.FTP;
050 import org.apache.commons.net.ftp.FTPClient;
051
052 import ucar.unidata.data.BadDataException;
053 import ucar.unidata.data.DataSourceDescriptor;
054 import ucar.unidata.util.DateUtil;
055 import ucar.unidata.util.IOUtil;
056 import ucar.unidata.util.StringUtil;
057 import visad.DateTime;
058 import visad.Real;
059 import visad.RealType;
060 import visad.VisADException;
061 import visad.georef.EarthLocation;
062 import visad.georef.EarthLocationLite;
063
064 /**
065 */
066 public class AtcfStormDataSource extends StormDataSource {
067
068 /** _more_ */
069 private int BASEIDX = 0;
070
071 /** _more_ */
072 private int IDX_BASIN = BASEIDX++;
073
074 /** _more_ */
075 private int IDX_CY = BASEIDX++;
076
077 /** _more_ */
078 private int IDX_YYYYMMDDHH = BASEIDX++;
079
080 /** _more_ */
081 private int IDX_TECHNUM = BASEIDX++;
082
083 /** _more_ */
084 private int IDX_TECH = BASEIDX++;
085
086 /** _more_ */
087 private int IDX_TAU = BASEIDX++;
088
089 /** _more_ */
090 private int IDX_LAT = BASEIDX++;
091
092 /** _more_ */
093 private int IDX_LON = BASEIDX++;
094
095 /** _more_ */
096 private int IDX_VMAX = BASEIDX++;
097
098 /** _more_ */
099 private int IDX_MSLP = BASEIDX++;
100
101 /** _more_ */
102 private int IDX_TY = BASEIDX++;
103
104 /** _more_ */
105 private int IDX_RAD = BASEIDX++;
106
107 /** _more_ */
108 private int IDX_WINDCODE = BASEIDX++;
109
110 /** _more_ */
111 private int IDX_RAD1 = BASEIDX++;
112
113 /** _more_ */
114 private int IDX_RAD2 = BASEIDX++;
115
116 /** _more_ */
117 private int IDX_RAD3 = BASEIDX++;
118
119 /** _more_ */
120 private int IDX_RAD4 = BASEIDX++;
121
122 /** _more_ */
123 private int IDX_RADP = BASEIDX++;
124
125 /** _more_ */
126 private int IDX_RRP = BASEIDX++;
127
128 /** _more_ */
129 private int IDX_MRD = BASEIDX++;
130
131 /** _more_ */
132 private int IDX_GUSTS = BASEIDX++;
133
134 /** _more_ */
135 private int IDX_EYE = BASEIDX++;
136
137 /** _more_ */
138 private int IDX_SUBREGION = BASEIDX++;
139
140 /** _more_ */
141 private int IDX_MAXSEAS = BASEIDX++;
142
143 /** _more_ */
144 private int IDX_INITIALS = BASEIDX++;
145
146 /** _more_ */
147 private int IDX_DIR = BASEIDX++;
148
149 /** _more_ */
150 private int IDX_SPEED = BASEIDX++;
151
152 /** _more_ */
153 private int IDX_STORMNAME = BASEIDX++;
154
155 /** _more_ */
156 private int IDX_DEPTH = BASEIDX++;
157
158 /** _more_ */
159 private int IDX_SEAS = BASEIDX++;
160
161 /** _more_ */
162 private int IDX_SEASCODE = BASEIDX++;
163
164 /** _more_ */
165 private int IDX_SEAS1 = BASEIDX++;
166
167 /** _more_ */
168 private int IDX_SEAS2 = BASEIDX++;
169
170 /** _more_ */
171 private int IDX_SEAS3 = BASEIDX++;
172
173 /** _more_ */
174 private int IDX_SEAS4 = BASEIDX++;
175
176 /** _more_ */
177 private static final String PREFIX_ANALYSIS = "a";
178
179 /** _more_ */
180 private static final String PREFIX_BEST = "b";
181
182 /** _more_ */
183 private static final String WAY_BEST = "BEST";
184
185 /** _more_ */
186 private static final String WAY_CARQ = "CARQ";
187
188 /** _more_ */
189 private static final String WAY_WRNG = "WRNG";
190
191 /** _more_ */
192 private static String DEFAULT_PATH = "ftp://anonymous:password@ftp.nhc.noaa.gov/atcf";
193
194 /** _more_ */
195 private String path;
196
197 /** _more_ */
198 private List<StormInfo> stormInfos;
199
200 /** _more_ */
201 private StormTrackCollection localTracks;
202
203 /**
204 * _more_
205 *
206 * @throws Exception
207 * _more_
208 */
209 public AtcfStormDataSource() throws Exception {
210 }
211
212 /**
213 * _more_
214 *
215 * @return _more_
216 */
217 public String getFullDescription() {
218 return "ATCF Data Source<br>Path:" + path;
219 }
220
221 /**
222 * _more_
223 *
224 * @param descriptor
225 * _more_
226 * @param url
227 * _more_
228 * @param properties
229 * _more_
230 */
231 public AtcfStormDataSource(DataSourceDescriptor descriptor, String url,
232 Hashtable properties) {
233 super(descriptor, "ATCF Storm Data", "ATCF Storm Data", properties);
234 if ((url == null) || (url.trim().length() == 0)
235 || url.trim().equalsIgnoreCase("default")) {
236 url = DEFAULT_PATH;
237 }
238 path = url;
239 }
240
241 /**
242 * _more_
243 *
244 * @return _more_
245 */
246 public String getId() {
247 return "atcf";
248 }
249
250 /**
251 * _more_
252 *
253 * @param suffix
254 * _more_
255 *
256 * @return _more_
257 */
258 private String getFullPath(String suffix) {
259 return path + "/" + suffix;
260 }
261
262 /**
263 * _more_
264 */
265 protected void initializeStormData() {
266 try {
267 incrOutstandingGetDataCalls();
268 stormInfos = new ArrayList<StormInfo>();
269 if (path.toLowerCase().endsWith(".atcf")
270 || path.toLowerCase().endsWith(".gz")
271 || path.toLowerCase().endsWith(".dat")) {
272 String name = IOUtil.stripExtension(IOUtil.getFileTail(path));
273 StormInfo si = new StormInfo(name, new DateTime(new Date()));
274 stormInfos.add(si);
275 localTracks = new StormTrackCollection();
276 readTracks(si, localTracks, path, null, true);
277 List<StormTrack> trackList = localTracks.getTracks();
278
279 if (trackList.size() > 0) {
280 si.setStartTime(trackList.get(0).getStartTime());
281 }
282 return;
283 }
284
285 byte[] techs = readFile(getFullPath("nhc_techlist.dat"), true);
286 if (techs != null) {
287 /*
288 * NUM TECH ERRS RETIRED COLOR DEFAULTS INT-DEFS RADII-DEFS
289 * LONG-NAME 00 CARQ 0 0 0 0 0 1 Combined ARQ Position 00 WRNG 0
290 * 0 0 0 0 1 Warning
291 */
292 int cnt = 0;
293 for (String line : StringUtil.split(new String(techs), "\n",
294 true, true)) {
295 if (cnt++ == 0) {
296 continue;
297 }
298 if (line.length() > 67) {
299 String id = line.substring(3, 10).trim();
300 String name = line.substring(67).trim();
301 // System.out.println (id + ":" +name);
302 getWay(id, name);
303 }
304 }
305 }
306
307 // byte[] bytes = readFile(getFullPath("archive/storm.table"),
308 byte[] bytes = readFile(getFullPath("index/storm_list.txt"), false);
309 String stormTable = new String(bytes);
310 List lines = StringUtil.split(stormTable, "\n", true, true);
311
312 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH");
313 fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
314 for (int i = 0; i < lines.size(); i++) {
315 String line = (String) lines.get(i);
316 List toks = StringUtil.split(line, ",", true);
317 String name = (String) toks.get(0);
318 String basin = (String) toks.get(1);
319 String number = (String) toks.get(7);
320 String year = (String) toks.get(8);
321 int y = new Integer(year).intValue();
322 String id = basin + "_" + number + "_" + year;
323 if (name.equals("UNNAMED")) {
324 name = id;
325 }
326 String dttm = (String) toks.get(11);
327 Date date = fmt.parse(dttm);
328 StormInfo si = new StormInfo(id, name, basin, number,
329 new DateTime(date));
330 stormInfos.add(si);
331
332 }
333 } catch (Exception exc) {
334 logException("Error initializing ATCF data", exc);
335 } finally {
336 decrOutstandingGetDataCalls();
337 }
338 }
339
340 /**
341 * _more_
342 *
343 * @return _more_
344 */
345 public List<StormInfo> getStormInfos() {
346 return stormInfos;
347 }
348
349 /**
350 * _more_
351 *
352 * @param s
353 * _more_
354 *
355 * @return _more_
356 */
357 private double getDouble(String s) {
358 if (s == null) {
359 return Double.NaN;
360 }
361 if (s.length() == 0) {
362 return Double.NaN;
363 }
364 return new Double(s).doubleValue();
365 }
366
367 /**
368 * _more_
369 *
370 * @throws VisADException
371 * _more_
372 */
373 protected void initParams() throws VisADException {
374 super.initParams();
375 if (obsParams == null) {
376 obsParams = new StormParam[] { PARAM_STORMCATEGORY,
377 PARAM_MINPRESSURE, PARAM_MAXWINDSPEED_KTS };
378
379 }
380 }
381
382 /**
383 * _more_
384 *
385 * @param stormInfo
386 * _more_
387 * @param tracks
388 * _more_
389 * @param trackFile
390 * _more_
391 * @param waysToUse
392 * _more_
393 * @param throwError
394 * _more_
395 *
396 *
397 * @return _more_
398 * @throws Exception
399 * _more_
400 */
401 private boolean readTracks(StormInfo stormInfo,
402 StormTrackCollection tracks, String trackFile,
403 Hashtable<String, Boolean> waysToUse, boolean throwError)
404 throws Exception {
405
406 long t1 = System.currentTimeMillis();
407 byte[] bytes = readFile(trackFile, true);
408 long t2 = System.currentTimeMillis();
409 // System.err.println("read time:" + (t2 - t1));
410 boolean isZip = trackFile.endsWith(".gz");
411 if ((bytes == null) && isZip) {
412 String withoutGZ = trackFile.substring(0, trackFile.length() - 3);
413 bytes = readFile(withoutGZ, true);
414 isZip = false;
415 }
416
417 if (bytes == null) {
418 if (throwError) {
419 throw new BadDataException("Unable to read track file:"
420 + trackFile);
421 }
422 return false;
423 }
424
425 if (isZip) {
426 GZIPInputStream zin = new GZIPInputStream(new ByteArrayInputStream(
427 bytes));
428 bytes = IOUtil.readBytes(zin);
429 zin.close();
430 }
431 GregorianCalendar convertCal = new GregorianCalendar(
432 DateUtil.TIMEZONE_GMT);
433 convertCal.clear();
434
435 String trackData = new String(bytes);
436 List lines = StringUtil.split(trackData, "\n", true, true);
437 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH");
438 fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
439 Hashtable trackMap = new Hashtable();
440 Real altReal = new Real(RealType.Altitude, 0);
441 // System.err.println("obs:" + lines.size());
442 /*
443 * Hashtable okWays = new Hashtable(); okWays.put(WAY_CARQ, "");
444 * okWays.put(WAY_WRNG, ""); okWays.put(WAY_BEST, ""); okWays.put("ETA",
445 * ""); okWays.put("NGX", ""); okWays.put("BAMS", "");
446 */
447 Hashtable seenDate = new Hashtable();
448 initParams();
449 int xcnt = 0;
450 for (int i = 0; i < lines.size(); i++) {
451 String line = (String) lines.get(i);
452 if (i == 0) {
453 // System.err.println(line);
454 }
455 List toks = StringUtil.split(line, ",", true);
456 /*
457 * System.err.println(toks.size() + " " + BASEIDX);
458 * if(toks.size()<BASEIDX-1) { System.err.println("bad line:" +
459 * line); continue; } else { System.err.println("good line:" +
460 * line); }
461 */
462
463 // BASIN,CY,YYYYMMDDHH,TECHNUM,TECH,TAU,LatN/S,LonE/W,VMAX,MSLP,TY,RAD,WINDCODE,RAD1,RAD2,RAD3,RAD4,RADP,RRP,MRD,GUSTS,EYE,SUBREGION,MAXSEAS,INITIALS,DIR,SPEED,STORMNAME,DEPTH,SEAS,SEASCODE,SEAS1,SEAS2,SEAS3,SEAS4
464 // AL, 01, 2007050612, , BEST, 0, 355N, 740W, 35, 1012, EX, 34, NEQ,
465 // 0, 0, 0, 120,
466 // AL, 01, 2007050812, 01, CARQ, -24, 316N, 723W, 55, 0, DB, 34,
467 // AAA, 0, 0, 0, 0,
468
469 String dateString = (String) toks.get(IDX_YYYYMMDDHH);
470 String wayString = (String) toks.get(IDX_TECH);
471 // if (okWays.get(wayString) == null) {
472 // continue;
473 // }
474 boolean isBest = wayString.equals(WAY_BEST);
475 boolean isWarning = wayString.equals(WAY_WRNG);
476 boolean isCarq = wayString.equals(WAY_CARQ);
477
478 int category = ((IDX_TY < toks.size()) ? getCategory((String) toks
479 .get(IDX_TY)) : CATEGORY_XX);
480 if (category != CATEGORY_XX) {
481 // System.err.println("cat:" + category);
482 }
483
484 String fhour = (String) toks.get(IDX_TAU);
485 int forecastHour = new Integer(fhour).intValue();
486 // A hack - we've seen some atfc files that have a 5 character
487 // forecast hour
488 // right padded with "00", eg., 01200
489 if ((fhour.length() == 5) && (forecastHour > 100)) {
490 forecastHour = forecastHour / 100;
491 }
492
493 if (isWarning || isCarq) {
494 forecastHour = -forecastHour;
495 }
496
497 // Check for unique dates for this way
498 String dttmkey = wayString + "_" + dateString + "_" + forecastHour;
499 if (seenDate.get(dttmkey) != null) {
500 continue;
501 }
502 seenDate.put(dttmkey, dttmkey);
503
504 Date dttm = fmt.parse(dateString);
505 convertCal.setTime(dttm);
506 String key;
507 Way way = getWay(wayString, null);
508 if (!isBest && (waysToUse != null) && (waysToUse.size() > 0)
509 && (waysToUse.get(wayString) == null)) {
510 continue;
511 }
512
513 if (isBest) {
514 key = wayString;
515 } else {
516 key = wayString + "_" + dateString;
517 convertCal.add(Calendar.HOUR_OF_DAY, forecastHour);
518 }
519 dttm = convertCal.getTime();
520 StormTrack track = (StormTrack) trackMap.get(key);
521 if (track == null) {
522 way = (isBest ? Way.OBSERVATION : way);
523 track = new StormTrack(stormInfo, addWay(way), new DateTime(
524 dttm), obsParams);
525 trackMap.put(key, track);
526 tracks.addTrack(track);
527 }
528 String latString = (String) toks.get(IDX_LAT);
529 String lonString = (String) toks.get(IDX_LON);
530 String t = latString + " " + lonString;
531
532 boolean south = latString.endsWith("S");
533 boolean west = lonString.endsWith("W");
534 double latitude = Double.parseDouble(latString.substring(0,
535 latString.length() - 1)) / 10.0;
536 double longitude = Double.parseDouble(lonString.substring(0,
537 lonString.length() - 1)) / 10.0;
538 if (south) {
539 latitude = -latitude;
540 }
541 if (west) {
542 longitude = -longitude;
543 }
544
545 EarthLocation elt = new EarthLocationLite(new Real(
546 RealType.Latitude, latitude), new Real(RealType.Longitude,
547 longitude), altReal);
548
549 List<Real> attributes = new ArrayList<Real>();
550
551 double windspeed = ((IDX_VMAX < toks.size()) ? getDouble((String) toks
552 .get(IDX_VMAX))
553 : Double.NaN);
554 double pressure = ((IDX_MSLP < toks.size()) ? getDouble((String) toks
555 .get(IDX_MSLP))
556 : Double.NaN);
557 attributes.add(PARAM_STORMCATEGORY.getReal((double) category));
558 attributes.add(PARAM_MINPRESSURE.getReal(pressure));
559 attributes.add(PARAM_MAXWINDSPEED_KTS.getReal(windspeed));
560
561 StormTrackPoint stp = new StormTrackPoint(elt, new DateTime(dttm),
562 forecastHour, attributes);
563
564 track.addPoint(stp);
565 }
566 return true;
567 }
568
569 /**
570 * _more_
571 *
572 * @return _more_
573 */
574 public String getWayName() {
575 return "Tech";
576 }
577
578 /**
579 * _more_
580 *
581 * @param stormInfo
582 * _more_
583 * @param waysToUse
584 * _more_
585 * @param observationWay
586 * _more_
587 *
588 * @return _more_
589 *
590 * @throws Exception
591 * _more_
592 */
593 public StormTrackCollection getTrackCollectionInner(StormInfo stormInfo,
594 Hashtable<String, Boolean> waysToUse, Way observationWay)
595 throws Exception {
596 if (localTracks != null) {
597 return localTracks;
598 }
599
600 long t1 = System.currentTimeMillis();
601 StormTrackCollection tracks = new StormTrackCollection();
602
603 String trackFile;
604 boolean justObs = (waysToUse != null) && (waysToUse.size() == 1)
605 && (waysToUse.get(Way.OBSERVATION.toString()) != null);
606 int nowYear = new GregorianCalendar(DateUtil.TIMEZONE_GMT)
607 .get(Calendar.YEAR);
608 int stormYear = getYear(stormInfo.getStartTime());
609 // If its the current year then its in the aid_public dir
610 String aSubDir = ((stormYear == nowYear) ? "aid_public"
611 : ("archive/" + stormYear));
612 String bSubDir = ((stormYear == nowYear) ? "btk"
613 : ("archive/" + stormYear));
614 if (!justObs) {
615 trackFile = getFullPath(aSubDir + "/" + PREFIX_ANALYSIS
616 + stormInfo.getBasin().toLowerCase()
617 + stormInfo.getNumber() + stormYear + ".dat.gz");
618 // What we think might be in the archive might actually be the last
619 // year
620 // and they haven't moved it into the archive
621 try {
622 readTracks(stormInfo, tracks, trackFile, waysToUse, true);
623 } catch (BadDataException bde) {
624 if (!aSubDir.equals("aid_public")) {
625 try {
626 trackFile = getFullPath("aid_public/" + PREFIX_ANALYSIS
627 + stormInfo.getBasin().toLowerCase()
628 + stormInfo.getNumber() + stormYear + ".dat.gz");
629 readTracks(stormInfo, tracks, trackFile, waysToUse,
630 true);
631 } catch (BadDataException bde2) {
632 System.err.println("Failed reading 'A' file for storm:"
633 + stormInfo + " file:" + trackFile);
634 }
635 }
636 // System.err.println("Failed reading 'A' file for storm:" +
637 // stormInfo+" file:" + trackFile);
638 }
639 }
640 // Now read the b"est file
641 trackFile = getFullPath(bSubDir + "/" + PREFIX_BEST
642 + stormInfo.getBasin().toLowerCase() + stormInfo.getNumber()
643 + stormYear + ".dat.gz");
644 try {
645 readTracks(stormInfo, tracks, trackFile, null, true);
646 } catch (BadDataException bde) {
647 if (!bSubDir.equals("btk")) {
648 try {
649 trackFile = getFullPath("btk/" + PREFIX_BEST
650 + stormInfo.getBasin().toLowerCase()
651 + stormInfo.getNumber() + stormYear + ".dat.gz");
652 readTracks(stormInfo, tracks, trackFile, null, true);
653 } catch (BadDataException bde2) {
654 System.err.println("Failed reading 'B' file for storm:"
655 + stormInfo + " file:" + trackFile);
656 }
657
658 }
659 // System.err.println("Failed reading 'B' file for storm:" +
660 // stormInfo+" file:" + trackFile);
661 }
662 long t2 = System.currentTimeMillis();
663 // System.err.println("time: " + (t2 - t1));
664
665 return tracks;
666 }
667
668 /**
669 * Set the Directory property.
670 *
671 * @param value
672 * The new value for Directory
673 */
674 public void setPath(String value) {
675 path = value;
676 }
677
678 /**
679 * Get the Directory property.
680 *
681 * @return The Directory
682 */
683 public String getPath() {
684 return path;
685 }
686
687 /**
688 * _more_
689 *
690 * @param file
691 * _more_
692 * @param ignoreErrors
693 * _more_
694 *
695 * @return _more_
696 *
697 * @throws Exception
698 * _more_
699 */
700 private byte[] readFile(String file, boolean ignoreErrors) throws Exception {
701 if (new File(file).exists()) {
702 return IOUtil.readBytes(IOUtil.getInputStream(file, getClass()));
703 }
704 if (!file.startsWith("ftp:")) {
705 if (ignoreErrors) {
706 return null;
707 }
708 throw new FileNotFoundException("Could not read file: " + file);
709 }
710
711 URL url = new URL(file);
712 FTPClient ftp = new FTPClient();
713 try {
714 ftp.connect(url.getHost());
715 ftp.login("anonymous", "password");
716 ftp.setFileType(FTP.IMAGE_FILE_TYPE);
717 ftp.enterLocalPassiveMode();
718 ByteArrayOutputStream bos = new ByteArrayOutputStream();
719 if (ftp.retrieveFile(url.getPath(), bos)) {
720 return bos.toByteArray();
721 } else {
722 throw new IOException("Unable to retrieve file:" + url);
723 }
724 } catch (org.apache.commons.net.ftp.FTPConnectionClosedException fcce) {
725 System.err.println("ftp error:" + fcce);
726 System.err.println(ftp.getReplyString());
727 if (!ignoreErrors) {
728 throw fcce;
729 }
730 return null;
731 } catch (Exception exc) {
732 if (!ignoreErrors) {
733 throw exc;
734 }
735 return null;
736 } finally {
737 try {
738 ftp.logout();
739 } catch (Exception exc) {
740 }
741 try {
742 ftp.disconnect();
743 } catch (Exception exc) {
744 }
745
746 }
747 }
748
749 }