001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2024 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas/ 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see https://www.gnu.org/licenses/. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.cyclone; 030 031import java.io.ByteArrayInputStream; 032import java.io.ByteArrayOutputStream; 033import java.io.File; 034import java.io.FileNotFoundException; 035import java.io.IOException; 036import java.net.URL; 037import java.text.SimpleDateFormat; 038import java.util.ArrayList; 039import java.util.Calendar; 040import java.util.Date; 041import java.util.GregorianCalendar; 042import java.util.Hashtable; 043import java.util.List; 044import java.util.TimeZone; 045import java.util.zip.GZIPInputStream; 046 047import org.apache.commons.net.ftp.FTP; 048import org.apache.commons.net.ftp.FTPClient; 049 050import ucar.unidata.data.BadDataException; 051import ucar.unidata.data.DataSourceDescriptor; 052import ucar.unidata.util.DateUtil; 053import ucar.unidata.util.IOUtil; 054import ucar.unidata.util.StringUtil; 055import visad.DateTime; 056import visad.Real; 057import visad.RealType; 058import visad.VisADException; 059import visad.georef.EarthLocation; 060import visad.georef.EarthLocationLite; 061 062/** 063 */ 064public class AtcfStormDataSource extends StormDataSource { 065 066 /** _more_ */ 067 private int BASEIDX = 0; 068 069 /** _more_ */ 070 private int IDX_BASIN = BASEIDX++; 071 072 /** _more_ */ 073 private int IDX_CY = BASEIDX++; 074 075 /** _more_ */ 076 private int IDX_YYYYMMDDHH = BASEIDX++; 077 078 /** _more_ */ 079 private int IDX_TECHNUM = BASEIDX++; 080 081 /** _more_ */ 082 private int IDX_TECH = BASEIDX++; 083 084 /** _more_ */ 085 private int IDX_TAU = BASEIDX++; 086 087 /** _more_ */ 088 private int IDX_LAT = BASEIDX++; 089 090 /** _more_ */ 091 private int IDX_LON = BASEIDX++; 092 093 /** _more_ */ 094 private int IDX_VMAX = BASEIDX++; 095 096 /** _more_ */ 097 private int IDX_MSLP = BASEIDX++; 098 099 /** _more_ */ 100 private int IDX_TY = BASEIDX++; 101 102 /** _more_ */ 103 private int IDX_RAD = BASEIDX++; 104 105 /** _more_ */ 106 private int IDX_WINDCODE = BASEIDX++; 107 108 /** _more_ */ 109 private int IDX_RAD1 = BASEIDX++; 110 111 /** _more_ */ 112 private int IDX_RAD2 = BASEIDX++; 113 114 /** _more_ */ 115 private int IDX_RAD3 = BASEIDX++; 116 117 /** _more_ */ 118 private int IDX_RAD4 = BASEIDX++; 119 120 /** _more_ */ 121 private int IDX_RADP = BASEIDX++; 122 123 /** _more_ */ 124 private int IDX_RRP = BASEIDX++; 125 126 /** _more_ */ 127 private int IDX_MRD = BASEIDX++; 128 129 /** _more_ */ 130 private int IDX_GUSTS = BASEIDX++; 131 132 /** _more_ */ 133 private int IDX_EYE = BASEIDX++; 134 135 /** _more_ */ 136 private int IDX_SUBREGION = BASEIDX++; 137 138 /** _more_ */ 139 private int IDX_MAXSEAS = BASEIDX++; 140 141 /** _more_ */ 142 private int IDX_INITIALS = BASEIDX++; 143 144 /** _more_ */ 145 private int IDX_DIR = BASEIDX++; 146 147 /** _more_ */ 148 private int IDX_SPEED = BASEIDX++; 149 150 /** _more_ */ 151 private int IDX_STORMNAME = BASEIDX++; 152 153 /** _more_ */ 154 private int IDX_DEPTH = BASEIDX++; 155 156 /** _more_ */ 157 private int IDX_SEAS = BASEIDX++; 158 159 /** _more_ */ 160 private int IDX_SEASCODE = BASEIDX++; 161 162 /** _more_ */ 163 private int IDX_SEAS1 = BASEIDX++; 164 165 /** _more_ */ 166 private int IDX_SEAS2 = BASEIDX++; 167 168 /** _more_ */ 169 private int IDX_SEAS3 = BASEIDX++; 170 171 /** _more_ */ 172 private int IDX_SEAS4 = BASEIDX++; 173 174 /** _more_ */ 175 private static final String PREFIX_ANALYSIS = "a"; 176 177 /** _more_ */ 178 private static final String PREFIX_BEST = "b"; 179 180 /** _more_ */ 181 private static final String WAY_BEST = "BEST"; 182 183 /** _more_ */ 184 private static final String WAY_CARQ = "CARQ"; 185 186 /** _more_ */ 187 private static final String WAY_WRNG = "WRNG"; 188 189 /** _more_ */ 190 private static String DEFAULT_PATH = "ftp://anonymous:password@ftp.nhc.noaa.gov/atcf"; 191 192 /** _more_ */ 193 private String path; 194 195 /** _more_ */ 196 private List<StormInfo> stormInfos; 197 198 /** _more_ */ 199 private StormTrackCollection localTracks; 200 201 /** 202 * _more_ 203 * 204 * @throws Exception 205 * _more_ 206 */ 207 public AtcfStormDataSource() throws Exception { 208 } 209 210 /** 211 * _more_ 212 * 213 * @return _more_ 214 */ 215 public String getFullDescription() { 216 return "ATCF Data Source<br>Path:" + path; 217 } 218 219 /** 220 * _more_ 221 * 222 * @param descriptor 223 * _more_ 224 * @param url 225 * _more_ 226 * @param properties 227 * _more_ 228 */ 229 public AtcfStormDataSource(DataSourceDescriptor descriptor, String url, 230 Hashtable properties) { 231 super(descriptor, "ATCF Storm Data", "ATCF Storm Data", properties); 232 if ((url == null) || (url.trim().length() == 0) 233 || url.trim().equalsIgnoreCase("default")) { 234 url = DEFAULT_PATH; 235 } 236 path = url; 237 } 238 239 /** 240 * _more_ 241 * 242 * @return _more_ 243 */ 244 public String getId() { 245 return "atcf"; 246 } 247 248 /** 249 * _more_ 250 * 251 * @param suffix 252 * _more_ 253 * 254 * @return _more_ 255 */ 256 private String getFullPath(String suffix) { 257 return path + "/" + suffix; 258 } 259 260 /** 261 * _more_ 262 */ 263 protected void initializeStormData() { 264 try { 265 incrOutstandingGetDataCalls(); 266 stormInfos = new ArrayList<StormInfo>(); 267 if (path.toLowerCase().endsWith(".atcf") 268 || path.toLowerCase().endsWith(".gz") 269 || path.toLowerCase().endsWith(".dat")) { 270 String name = IOUtil.stripExtension(IOUtil.getFileTail(path)); 271 StormInfo si = new StormInfo(name, new DateTime(new Date())); 272 stormInfos.add(si); 273 localTracks = new StormTrackCollection(); 274 readTracks(si, localTracks, path, null, true); 275 List<StormTrack> trackList = localTracks.getTracks(); 276 277 if (trackList.size() > 0) { 278 si.setStartTime(trackList.get(0).getStartTime()); 279 } 280 return; 281 } 282 283 byte[] techs = readFile(getFullPath("nhc_techlist.dat"), true); 284 if (techs != null) { 285 /* 286 * NUM TECH ERRS RETIRED COLOR DEFAULTS INT-DEFS RADII-DEFS 287 * LONG-NAME 00 CARQ 0 0 0 0 0 1 Combined ARQ Position 00 WRNG 0 288 * 0 0 0 0 1 Warning 289 */ 290 int cnt = 0; 291 for (String line : StringUtil.split(new String(techs), "\n", 292 true, true)) { 293 if (cnt++ == 0) { 294 continue; 295 } 296 if (line.length() > 67) { 297 String id = line.substring(3, 10).trim(); 298 String name = line.substring(67).trim(); 299 // System.out.println (id + ":" +name); 300 getWay(id, name); 301 } 302 } 303 } 304 305 // byte[] bytes = readFile(getFullPath("archive/storm.table"), 306 byte[] bytes = readFile(getFullPath("index/storm_list.txt"), false); 307 String stormTable = new String(bytes); 308 List lines = StringUtil.split(stormTable, "\n", true, true); 309 310 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH"); 311 fmt.setTimeZone(TimeZone.getTimeZone("UTC")); 312 for (int i = 0; i < lines.size(); i++) { 313 String line = (String) lines.get(i); 314 List toks = StringUtil.split(line, ",", true); 315 String name = (String) toks.get(0); 316 String basin = (String) toks.get(1); 317 String number = (String) toks.get(7); 318 String year = (String) toks.get(8); 319 int y = new Integer(year).intValue(); 320 String id = basin + "_" + number + "_" + year; 321 if (name.equals("UNNAMED")) { 322 name = id; 323 } 324 String dttm = (String) toks.get(11); 325 Date date = fmt.parse(dttm); 326 StormInfo si = new StormInfo(id, name, basin, number, 327 new DateTime(date)); 328 stormInfos.add(si); 329 330 } 331 } catch (Exception exc) { 332 logException("Error initializing ATCF data", exc); 333 } finally { 334 decrOutstandingGetDataCalls(); 335 } 336 } 337 338 /** 339 * _more_ 340 * 341 * @return _more_ 342 */ 343 public List<StormInfo> getStormInfos() { 344 return stormInfos; 345 } 346 347 /** 348 * _more_ 349 * 350 * @param s 351 * _more_ 352 * 353 * @return _more_ 354 */ 355 private double getDouble(String s) { 356 if (s == null) { 357 return Double.NaN; 358 } 359 if (s.length() == 0) { 360 return Double.NaN; 361 } 362 return new Double(s).doubleValue(); 363 } 364 365 /** 366 * _more_ 367 * 368 * @throws VisADException 369 * _more_ 370 */ 371 protected void initParams() throws VisADException { 372 super.initParams(); 373 if (obsParams == null) { 374 obsParams = new StormParam[] { PARAM_STORMCATEGORY, 375 PARAM_MINPRESSURE, PARAM_MAXWINDSPEED_KTS }; 376 377 } 378 } 379 380 /** 381 * _more_ 382 * 383 * @param stormInfo 384 * _more_ 385 * @param tracks 386 * _more_ 387 * @param trackFile 388 * _more_ 389 * @param waysToUse 390 * _more_ 391 * @param throwError 392 * _more_ 393 * 394 * 395 * @return _more_ 396 * @throws Exception 397 * _more_ 398 */ 399 private boolean readTracks(StormInfo stormInfo, 400 StormTrackCollection tracks, String trackFile, 401 Hashtable<String, Boolean> waysToUse, boolean throwError) 402 throws Exception { 403 404 long t1 = System.currentTimeMillis(); 405 byte[] bytes = readFile(trackFile, true); 406 long t2 = System.currentTimeMillis(); 407 // System.err.println("read time:" + (t2 - t1)); 408 boolean isZip = trackFile.endsWith(".gz"); 409 if ((bytes == null) && isZip) { 410 String withoutGZ = trackFile.substring(0, trackFile.length() - 3); 411 bytes = readFile(withoutGZ, true); 412 isZip = false; 413 } 414 415 if (bytes == null) { 416 if (throwError) { 417 throw new BadDataException("Unable to read track file:" 418 + trackFile); 419 } 420 return false; 421 } 422 423 if (isZip) { 424 GZIPInputStream zin = new GZIPInputStream(new ByteArrayInputStream( 425 bytes)); 426 bytes = IOUtil.readBytes(zin); 427 zin.close(); 428 } 429 GregorianCalendar convertCal = new GregorianCalendar( 430 DateUtil.TIMEZONE_GMT); 431 convertCal.clear(); 432 433 String trackData = new String(bytes); 434 List lines = StringUtil.split(trackData, "\n", true, true); 435 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH"); 436 fmt.setTimeZone(TimeZone.getTimeZone("UTC")); 437 Hashtable trackMap = new Hashtable(); 438 Real altReal = new Real(RealType.Altitude, 0); 439 // System.err.println("obs:" + lines.size()); 440 /* 441 * Hashtable okWays = new Hashtable(); okWays.put(WAY_CARQ, ""); 442 * okWays.put(WAY_WRNG, ""); okWays.put(WAY_BEST, ""); okWays.put("ETA", 443 * ""); okWays.put("NGX", ""); okWays.put("BAMS", ""); 444 */ 445 Hashtable seenDate = new Hashtable(); 446 initParams(); 447 int xcnt = 0; 448 for (int i = 0; i < lines.size(); i++) { 449 String line = (String) lines.get(i); 450 if (i == 0) { 451 // System.err.println(line); 452 } 453 List toks = StringUtil.split(line, ",", true); 454 /* 455 * System.err.println(toks.size() + " " + BASEIDX); 456 * if(toks.size()<BASEIDX-1) { System.err.println("bad line:" + 457 * line); continue; } else { System.err.println("good line:" + 458 * line); } 459 */ 460 461 // BASIN,CY,YYYYMMDDHH,TECHNUM,TECH,TAU,LatN/S,LonE/W,VMAX,MSLP,TY,RAD,WINDCODE,RAD1,RAD2,RAD3,RAD4,RADP,RRP,MRD,GUSTS,EYE,SUBREGION,MAXSEAS,INITIALS,DIR,SPEED,STORMNAME,DEPTH,SEAS,SEASCODE,SEAS1,SEAS2,SEAS3,SEAS4 462 // AL, 01, 2007050612, , BEST, 0, 355N, 740W, 35, 1012, EX, 34, NEQ, 463 // 0, 0, 0, 120, 464 // AL, 01, 2007050812, 01, CARQ, -24, 316N, 723W, 55, 0, DB, 34, 465 // AAA, 0, 0, 0, 0, 466 467 String dateString = (String) toks.get(IDX_YYYYMMDDHH); 468 String wayString = (String) toks.get(IDX_TECH); 469 // if (okWays.get(wayString) == null) { 470 // continue; 471 // } 472 boolean isBest = wayString.equals(WAY_BEST); 473 boolean isWarning = wayString.equals(WAY_WRNG); 474 boolean isCarq = wayString.equals(WAY_CARQ); 475 476 int category = ((IDX_TY < toks.size()) ? getCategory((String) toks 477 .get(IDX_TY)) : CATEGORY_XX); 478 if (category != CATEGORY_XX) { 479 // System.err.println("cat:" + category); 480 } 481 482 String fhour = (String) toks.get(IDX_TAU); 483 int forecastHour = new Integer(fhour).intValue(); 484 // A hack - we've seen some atfc files that have a 5 character 485 // forecast hour 486 // right padded with "00", eg., 01200 487 if ((fhour.length() == 5) && (forecastHour > 100)) { 488 forecastHour = forecastHour / 100; 489 } 490 491 if (isWarning || isCarq) { 492 forecastHour = -forecastHour; 493 } 494 495 // Check for unique dates for this way 496 String dttmkey = wayString + "_" + dateString + "_" + forecastHour; 497 if (seenDate.get(dttmkey) != null) { 498 continue; 499 } 500 seenDate.put(dttmkey, dttmkey); 501 502 Date dttm = fmt.parse(dateString); 503 convertCal.setTime(dttm); 504 String key; 505 Way way = getWay(wayString, null); 506 if (!isBest && (waysToUse != null) && (waysToUse.size() > 0) 507 && (waysToUse.get(wayString) == null)) { 508 continue; 509 } 510 511 if (isBest) { 512 key = wayString; 513 } else { 514 key = wayString + "_" + dateString; 515 convertCal.add(Calendar.HOUR_OF_DAY, forecastHour); 516 } 517 dttm = convertCal.getTime(); 518 StormTrack track = (StormTrack) trackMap.get(key); 519 if (track == null) { 520 way = (isBest ? Way.OBSERVATION : way); 521 track = new StormTrack(stormInfo, addWay(way), new DateTime( 522 dttm), obsParams); 523 trackMap.put(key, track); 524 tracks.addTrack(track); 525 } 526 String latString = (String) toks.get(IDX_LAT); 527 String lonString = (String) toks.get(IDX_LON); 528 String t = latString + " " + lonString; 529 530 boolean south = latString.endsWith("S"); 531 boolean west = lonString.endsWith("W"); 532 double latitude = Double.parseDouble(latString.substring(0, 533 latString.length() - 1)) / 10.0; 534 double longitude = Double.parseDouble(lonString.substring(0, 535 lonString.length() - 1)) / 10.0; 536 if (south) { 537 latitude = -latitude; 538 } 539 if (west) { 540 longitude = -longitude; 541 } 542 543 EarthLocation elt = new EarthLocationLite(new Real( 544 RealType.Latitude, latitude), new Real(RealType.Longitude, 545 longitude), altReal); 546 547 List<Real> attributes = new ArrayList<Real>(); 548 549 double windspeed = ((IDX_VMAX < toks.size()) ? getDouble((String) toks 550 .get(IDX_VMAX)) 551 : Double.NaN); 552 double pressure = ((IDX_MSLP < toks.size()) ? getDouble((String) toks 553 .get(IDX_MSLP)) 554 : Double.NaN); 555 attributes.add(PARAM_STORMCATEGORY.getReal((double) category)); 556 attributes.add(PARAM_MINPRESSURE.getReal(pressure)); 557 attributes.add(PARAM_MAXWINDSPEED_KTS.getReal(windspeed)); 558 559 StormTrackPoint stp = new StormTrackPoint(elt, new DateTime(dttm), 560 forecastHour, attributes); 561 562 track.addPoint(stp); 563 } 564 return true; 565 } 566 567 /** 568 * _more_ 569 * 570 * @return _more_ 571 */ 572 public String getWayName() { 573 return "Tech"; 574 } 575 576 /** 577 * _more_ 578 * 579 * @param stormInfo 580 * _more_ 581 * @param waysToUse 582 * _more_ 583 * @param observationWay 584 * _more_ 585 * 586 * @return _more_ 587 * 588 * @throws Exception 589 * _more_ 590 */ 591 public StormTrackCollection getTrackCollectionInner(StormInfo stormInfo, 592 Hashtable<String, Boolean> waysToUse, Way observationWay) 593 throws Exception { 594 if (localTracks != null) { 595 return localTracks; 596 } 597 598 long t1 = System.currentTimeMillis(); 599 StormTrackCollection tracks = new StormTrackCollection(); 600 601 String trackFile; 602 boolean justObs = (waysToUse != null) && (waysToUse.size() == 1) 603 && (waysToUse.get(Way.OBSERVATION.toString()) != null); 604 int nowYear = new GregorianCalendar(DateUtil.TIMEZONE_GMT) 605 .get(Calendar.YEAR); 606 int stormYear = getYear(stormInfo.getStartTime()); 607 // If its the current year then its in the aid_public dir 608 String aSubDir = ((stormYear == nowYear) ? "aid_public" 609 : ("archive/" + stormYear)); 610 String bSubDir = ((stormYear == nowYear) ? "btk" 611 : ("archive/" + stormYear)); 612 if (!justObs) { 613 trackFile = getFullPath(aSubDir + "/" + PREFIX_ANALYSIS 614 + stormInfo.getBasin().toLowerCase() 615 + stormInfo.getNumber() + stormYear + ".dat.gz"); 616 // What we think might be in the archive might actually be the last 617 // year 618 // and they haven't moved it into the archive 619 try { 620 readTracks(stormInfo, tracks, trackFile, waysToUse, true); 621 } catch (BadDataException bde) { 622 if (!aSubDir.equals("aid_public")) { 623 try { 624 trackFile = getFullPath("aid_public/" + PREFIX_ANALYSIS 625 + stormInfo.getBasin().toLowerCase() 626 + stormInfo.getNumber() + stormYear + ".dat.gz"); 627 readTracks(stormInfo, tracks, trackFile, waysToUse, 628 true); 629 } catch (BadDataException bde2) { 630 System.err.println("Failed reading 'A' file for storm:" 631 + stormInfo + " file:" + trackFile); 632 } 633 } 634 // System.err.println("Failed reading 'A' file for storm:" + 635 // stormInfo+" file:" + trackFile); 636 } 637 } 638 // Now read the b"est file 639 trackFile = getFullPath(bSubDir + "/" + PREFIX_BEST 640 + stormInfo.getBasin().toLowerCase() + stormInfo.getNumber() 641 + stormYear + ".dat.gz"); 642 try { 643 readTracks(stormInfo, tracks, trackFile, null, true); 644 } catch (BadDataException bde) { 645 if (!bSubDir.equals("btk")) { 646 try { 647 trackFile = getFullPath("btk/" + PREFIX_BEST 648 + stormInfo.getBasin().toLowerCase() 649 + stormInfo.getNumber() + stormYear + ".dat.gz"); 650 readTracks(stormInfo, tracks, trackFile, null, true); 651 } catch (BadDataException bde2) { 652 System.err.println("Failed reading 'B' file for storm:" 653 + stormInfo + " file:" + trackFile); 654 } 655 656 } 657 // System.err.println("Failed reading 'B' file for storm:" + 658 // stormInfo+" file:" + trackFile); 659 } 660 long t2 = System.currentTimeMillis(); 661 // System.err.println("time: " + (t2 - t1)); 662 663 return tracks; 664 } 665 666 /** 667 * Set the Directory property. 668 * 669 * @param value 670 * The new value for Directory 671 */ 672 public void setPath(String value) { 673 path = value; 674 } 675 676 /** 677 * Get the Directory property. 678 * 679 * @return The Directory 680 */ 681 public String getPath() { 682 return path; 683 } 684 685 /** 686 * _more_ 687 * 688 * @param file 689 * _more_ 690 * @param ignoreErrors 691 * _more_ 692 * 693 * @return _more_ 694 * 695 * @throws Exception 696 * _more_ 697 */ 698 private byte[] readFile(String file, boolean ignoreErrors) throws Exception { 699 if (new File(file).exists()) { 700 return IOUtil.readBytes(IOUtil.getInputStream(file, getClass())); 701 } 702 if (!file.startsWith("ftp:")) { 703 if (ignoreErrors) { 704 return null; 705 } 706 throw new FileNotFoundException("Could not read file: " + file); 707 } 708 709 URL url = new URL(file); 710 FTPClient ftp = new FTPClient(); 711 try { 712 ftp.connect(url.getHost()); 713 ftp.login("anonymous", "password"); 714 ftp.setFileType(FTP.IMAGE_FILE_TYPE); 715 ftp.enterLocalPassiveMode(); 716 ByteArrayOutputStream bos = new ByteArrayOutputStream(); 717 if (ftp.retrieveFile(url.getPath(), bos)) { 718 return bos.toByteArray(); 719 } else { 720 throw new IOException("Unable to retrieve file:" + url); 721 } 722 } catch (org.apache.commons.net.ftp.FTPConnectionClosedException fcce) { 723 System.err.println("ftp error:" + fcce); 724 System.err.println(ftp.getReplyString()); 725 if (!ignoreErrors) { 726 throw fcce; 727 } 728 return null; 729 } catch (Exception exc) { 730 if (!ignoreErrors) { 731 throw exc; 732 } 733 return null; 734 } finally { 735 try { 736 ftp.logout(); 737 } catch (Exception exc) { 738 } 739 try { 740 ftp.disconnect(); 741 } catch (Exception exc) { 742 } 743 744 } 745 } 746 747}