001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2018 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see http://www.gnu.org/licenses. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.hydra; 030 031import java.io.File; 032import java.io.IOException; 033import java.util.ArrayList; 034import java.util.HashMap; 035import java.util.HashSet; 036import java.util.Iterator; 037import java.util.List; 038import java.util.Map; 039import java.util.Set; 040import java.util.StringTokenizer; 041 042import ucar.nc2.Attribute; 043import ucar.nc2.NetcdfFile; 044 045/** 046 * Utility class to support Joint Polar Satellite System (JPSS) functionality. 047 * Documentation referenced is from Suomi NPP Common Data Format Control Book. 048 * See: 049 * http://npp.gsfc.nasa.gov/documents.html 050 * 051 * @author tommyj 052 * 053 */ 054 055public abstract class JPSSUtilities { 056 057 public static final String JPSS_FIELD_SEPARATOR = "_"; 058 public static final int NASA_CREATION_DATE_INDEX = 28; 059 public static final int NOAA_CREATION_DATE_INDEX = 35; 060 061 // Flags to check for SIPS data V2.0.0 and higher 062 // Augments single fill value from prior versions 063 public static final String SIPS_BOWTIE_DELETED_FLAG = "Bowtie_Deleted"; 064 public static final String SIPS_FLAG_MEANINGS_ATTRIBUTE = "flag_meanings"; 065 public static final String SIPS_FLAG_VALUES_ATTRIBUTE = "flag_values"; 066 067 // This regular expression matches a Suomi NPP Data Product as defined by the 068 // NOAA spec in CDFCB-X Volume 1, Page 21 069 public static final String SUOMI_NPP_REGEX_NOAA = 070 // Product Id, Multiple (ex: VSSTO-GATMO-VSLTO) 071 "(\\w\\w\\w\\w\\w-)*" + 072 // Product Id, Single (ex: VSSTO) 073 "\\w\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 074 // Spacecraft Id (ex: npp) 075 "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 076 // Data Start Date (ex: dYYYYMMDD) 077 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 078 // Data Start Time (ex: tHHMMSSS) 079 "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 080 // Data Stop Time (ex: eHHMMSSS) 081 "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 082 // Orbit Number (ex: b00015) 083 "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 084 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 085 "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 086 // Origin (ex: navo) 087 "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 088 // Domain (ex: ops) 089 "\\w\\w\\w" + 090 // HDF5 suffix 091 ".h5"; 092 093 // This regular expression matches a Suomi NPP Data Product as defined by the 094 // NASA spec in TBD (XXX TJJ - find out where is this documented?) 095 public static final String SUOMI_NPP_REGEX_NASA = 096 // Product Id, Single (ex: VL1BM) 097 // NOTE: These are the only supported NASA data at this time 098 "(VL1BD|VL1BI|VL1BM)" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 099 // Platform - always Suomi NPP 100 "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 101 // Data Start Date (ex: dYYYYMMDD) 102 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 103 // Data Start Time (ex: tHHMM) 104 "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 105 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 106 "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" + 107 // NetCDF 4 suffix 108 ".nc"; 109 110 // This regular expression matches a Suomi NPP geolocation file as defined by the 111 // NASA spec in TBD 112 public static final String SUOMI_GEO_REGEX_NASA = 113 // Product Id, Single (ex: VGEOM) 114 // NOTE: This MUST match the list of product ids in static array in this file! 115 "(VGEOD|VGEOI|VGEOM)" + 116 JPSSUtilities.JPSS_FIELD_SEPARATOR + 117 // Platform - always Suomi NPP 118 "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 119 // Data Start Date (ex: dYYYYMMDD) 120 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 121 // Data Start Time (ex: tHHMM) 122 "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 123 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 124 "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" + 125 // NetCDF 4 suffix 126 ".nc"; 127 128 public static String[] validNASAVariableNames = { 129 "M01", 130 "M02", 131 "M03", 132 "M04", 133 "M05", 134 "M06", 135 "M07", 136 "M08", 137 "M09", 138 "M10", 139 "M11", 140 "M12", 141 "M13", 142 "M14", 143 "M15", 144 "M16", 145 "I01", 146 "I02", 147 "I03", 148 "I04", 149 "I05", 150 "DNB_observations" 151 }; 152 153 public static float[] ATMSChannelCenterFrequencies = { 154 23.8f, 155 31.4f, 156 50.3f, 157 51.76f, 158 52.8f, 159 53.596f, 160 54.40f, 161 54.94f, 162 55.50f, 163 57.29032f, 164 57.29033f, 165 57.29034f, 166 57.29035f, 167 57.29036f, 168 57.29037f, 169 88.20f, 170 165.5f, 171 183.3101f, 172 183.3102f, 173 183.3103f, 174 183.3104f, 175 183.3105f 176 }; 177 178 // the list of valid geolocation product ids 179 public static String[] geoProductIDs = { 180 "GATMO", 181 "GCRSO", 182 "GAERO", 183 "GCLDO", 184 "GDNBO", 185 "GNCCO", 186 "GIGTO", 187 "GIMGO", 188 "GITCO", 189 "GMGTO", 190 "GMODO", 191 "GMTCO", 192 "GNHFO", 193 "GOTCO", 194 "GOSCO", 195 "GONPO", 196 "GONCO", 197 "GCRIO", 198 "GATRO", 199 "IVMIM", 200 "VGEOD", 201 "VGEOI", 202 "VGEOM", 203 "VMUGE" 204 }; 205 206 // This regular expression matches a Suomi NPP geolocation granule, see 207 // NOAA spec in CDFCB-X Volume 1, Page 21 208 public static final String SUOMI_GEO_REGEX_NOAA = 209 // Geo Id, Single (ex: GMODO) 210 // NOTE: This MUST match the list of product ids in static array above! 211 "(GATMO|GCRSO|GAERO|GCLDO|GDNBO|GNCCO|GIGTO|GIMGO|GITCO|" + 212 "GMGTO|GMODO|GMTCO|GNHFO|GOTCO|GOSCO|GONPO|GONCO|GCRIO|GATRO|IVMIM|VMUGE)" + 213 JPSSUtilities.JPSS_FIELD_SEPARATOR + 214 // Spacecraft Id (ex: npp) 215 "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 216 // Data Start Date (ex: dYYYYMMDD) 217 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 218 // Data Start Time (ex: tHHMMSSS) 219 "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 220 // Data Stop Time (ex: eHHMMSSS) 221 "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 222 // Orbit Number (ex: b00015) 223 "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 224 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 225 "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 226 // Origin (ex: navo) 227 "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 228 // Domain (ex: ops) 229 "\\w\\w\\w" + 230 // HDF5 suffix 231 ".h5"; 232 233 /** 234 * Determine if the input variable name is a valid NASA product 235 * 236 * @param varName Variable name to validate. 237 * 238 * @return {@code true} if {@code varName} passes checks. 239 */ 240 241 public static boolean isValidNASA(String varName) { 242 boolean isValid = false; 243 if (varName == null) return isValid; 244 for (String s : validNASAVariableNames) { 245 if (s.equals(varName)) { 246 isValid = true; 247 break; 248 } 249 } 250 return isValid; 251 } 252 253 /** 254 * Determine if the set if filenames constitutes contiguous SNPP granules 255 * of the same geographic coverage. 256 * 257 * @param fileList List of files to validate. 258 * 259 * @return {@code true} if {@code fileList} passes checks. 260 */ 261 262 public static boolean isValidSet(List fileList) { 263 264 // map with filename from start date through orbit will be used for comparisons 265 Map<String, List<String>> metadataMap = new HashMap<String, List<String>>(); 266 267 // Pass 1, populate the list of products selected, and empty maps 268 for (Object o : fileList) { 269 String filename = (String) o; 270 // start at last path separator to clip off absolute paths 271 int lastSeparator = filename.lastIndexOf(File.separatorChar); 272 // NOAA style products 273 if (filename.endsWith(".h5")) { 274 // products go to first underscore, see regex above for more detail 275 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 276 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 277 if (! metadataMap.containsKey(prodStr)) { 278 List<String> l = new ArrayList<String>(); 279 metadataMap.put(prodStr, l); 280 } 281 } 282 // NASA style products 283 if (filename.endsWith(".nc")) { 284 // products end at first underscore, see regex above for more detail 285 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 286 int secondUnderscore = filename.indexOf("_", firstUnderscore + 1); 287 String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore); 288 if (! metadataMap.containsKey(prodStr)) { 289 List<String> l = new ArrayList<String>(); 290 metadataMap.put(prodStr, l); 291 } 292 } 293 } 294 295 // Pass 2, build up the lists of meta data strings and full filenames 296 for (Object o : fileList) { 297 String filename = (String) o; 298 // start at last path separator to clip off absolute paths 299 int lastSeparator = filename.lastIndexOf(File.separatorChar); 300 301 // NOAA style products 302 if (filename.endsWith(".h5")) { 303 // products go to first underscore, see regex above for more detail 304 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 305 // this is the key for the maps 306 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 307 // this is the value for the meta data map - start time through orbit 308 String metaStr = filename.substring(firstUnderscore + 1, firstUnderscore + 39); 309 // get the appropriate list, add the new value 310 List<String> l = (List<String>) metadataMap.get(prodStr); 311 l.add(metaStr); 312 metadataMap.put(prodStr, l); 313 } 314 315 // NASA style products 316 if (filename.endsWith(".nc")) { 317 // products end at first underscore, see regex above for more detail 318 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 319 int secondUnderscore = filename.indexOf("_", firstUnderscore + 1); 320 // this is the key for the maps 321 String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore); 322 // this is the value for the meta data map - date and time 323 int dateIndex = filename.indexOf("_d"); 324 int creationIndex = filename.indexOf("_c"); 325 String metaStr = filename.substring(dateIndex + 1, creationIndex); 326 // get the appropriate list, add the new value 327 List<String> l = (List<String>) metadataMap.get(prodStr); 328 l.add(metaStr); 329 metadataMap.put(prodStr, l); 330 } 331 } 332 333 // loop over metadata map, every list much match the one for ALL other products 334 Set<String> s = metadataMap.keySet(); 335 Iterator iterator = s.iterator(); 336 List prvList = null; 337 while (iterator.hasNext()) { 338 String key = (String) iterator.next(); 339 List l = (List) metadataMap.get(key); 340 for (int i = 0; i < l.size(); i++) { 341 if (prvList != null) { 342 if (! l.equals(prvList)) return false; 343 } 344 } 345 prvList = l; 346 } 347 348 return true; 349 } 350 351 /** 352 * Replace last substring within input string which matches the input with the 353 * provided replacement string. 354 * 355 * @param string 356 * @param substring 357 * @param replacestr 358 * @return 359 */ 360 361 public static String replaceLast(String string, String substring, String replacestr) { 362 // Sanity check on input 363 if (string == null) return null; 364 if ((substring == null) || (replacestr == null)) return string; 365 366 int index = string.lastIndexOf(substring); 367 368 // substring not present 369 if (index == -1) 370 return string; 371 // it's there, swap it 372 return string.substring(0, index) + replacestr 373 + string.substring(index + substring.length()); 374 } 375 376 /** 377 * Determine if a set if filenames which constitutes contiguous SNPP 378 * granules of various products all share the same geolocation data type. 379 * 380 * @param fileList List of files to validate. 381 * @param directory Used when {@literal "GEO"} is not embedded within one 382 * of {@code fileList}. 383 * 384 * @return {@code true} if {@code fileList} passes checks. 385 */ 386 387 public static boolean hasCommonGeo(List fileList, File directory) { 388 Set<String> s = new HashSet<String>(); 389 boolean isCombinedProduct = false; 390 391 // loop through all filenames provided 392 for (Object o : fileList) { 393 isCombinedProduct = false; 394 String filename = (String) o; 395 396 // check the case where GEO is embedded in the data granules 397 int lastSeparator = filename.lastIndexOf(File.separatorChar); 398 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 399 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 400 StringTokenizer st = new StringTokenizer(prodStr, "-"); 401 while (st.hasMoreTokens()) { 402 String singleProd = st.nextToken(); 403 for (int i = 0; i < JPSSUtilities.geoProductIDs.length; i++) { 404 if (singleProd.equals(JPSSUtilities.geoProductIDs[i])) { 405 s.add(singleProd); 406 isCombinedProduct = true; 407 break; 408 } 409 } 410 } 411 // GEO not embedded in file, need to see which GEO file is 412 // referenced in the global attribute 413 if (! isCombinedProduct) { 414 try { 415 String fileFullPath = directory.getAbsolutePath() + File.separator + filename; 416 NetcdfFile ncfile = NetcdfFile.open(fileFullPath); 417 Attribute a = ncfile.findGlobalAttribute("N_GEO_Ref"); 418 if (a != null) { 419 String geoFromAttr = a.getStringValue().substring(0, 5); 420 s.add(geoFromAttr); 421 } 422 ncfile.close(); 423 } catch (IOException ioe) { 424 ioe.printStackTrace(); 425 } 426 } 427 } 428 429 // if the products chosen utilize multiple GEO types, fail the selection 430 if (s.size() > 1) return false; 431 return true; 432 } 433 434}