001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2024 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas/ 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see https://www.gnu.org/licenses/. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.hydra; 030 031import java.io.File; 032import java.io.IOException; 033import java.util.ArrayList; 034import java.util.HashMap; 035import java.util.HashSet; 036import java.util.Iterator; 037import java.util.List; 038import java.util.Map; 039import java.util.Set; 040import java.util.StringTokenizer; 041 042import org.slf4j.Logger; 043import org.slf4j.LoggerFactory; 044import ucar.nc2.Attribute; 045import ucar.nc2.NetcdfFile; 046 047/** 048 * Utility class to support Joint Polar Satellite System (JPSS) functionality. 049 * Documentation referenced is from Suomi NPP Common Data Format Control Book. 050 * See: 051 * http://npp.gsfc.nasa.gov/documents.html 052 * 053 * @author tommyj 054 * 055 */ 056 057public abstract class JPSSUtilities { 058 059 private static final Logger logger = 060 LoggerFactory.getLogger(JPSSUtilities.class); 061 062 public static final String JPSS_FIELD_SEPARATOR = "_"; 063 public static final int NASA_CREATION_DATE_INDEX = 28; 064 public static final int NOAA_CREATION_DATE_INDEX = 35; 065 066 // Flags to check for SIPS data V2.0.0 and higher 067 // Augments single fill value from prior versions 068 public static final String SIPS_BOWTIE_DELETED_FLAG = "Bowtie_Deleted"; 069 public static final String SIPS_FLAG_MEANINGS_ATTRIBUTE = "flag_meanings"; 070 public static final String SIPS_FLAG_VALUES_ATTRIBUTE = "flag_values"; 071 072 // This regular expression matches a Suomi NPP Data Product as defined by the 073 // NOAA spec in CDFCB-X Volume 1, Page 21 074 public static final String SUOMI_NPP_REGEX_NOAA = 075 // Product Id, Multiple (ex: VSSTO-GATMO-VSLTO) 076 "(\\w\\w\\w\\w\\w-)*" + 077 // Product Id, Single (ex: VSSTO) 078 "\\w\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 079 // Spacecraft Id (ex: npp) 080 "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 081 // Data Start Date (ex: dYYYYMMDD) 082 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 083 // Data Start Time (ex: tHHMMSSS) 084 "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 085 // Data Stop Time (ex: eHHMMSSS) 086 "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 087 // Orbit Number (ex: b00015) 088 "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 089 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 090 "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 091 // Origin (ex: navo) 092 "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 093 // Domain (ex: ops) 094 "\\w\\w\\w" + 095 // HDF5 suffix 096 ".h5"; 097 098 // This regular expression matches a VIIRS Enterprise EDR 099 public static final String JPSS_REGEX_ENTERPRISE_EDR = 100 // Product Id, Single (ex: JRR-CloudPhase) 101 "(\\w|-)*" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 102 // Version (e.g v2r3) 103 "v\\dr\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 104 // Origin (ex: npp) 105 "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 106 // Data Start Date/Time (ex: sYYYYMMDDHHMMSSS) 107 "s20[1-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 108 // Data End Date/Time (ex: eYYYYMMDDHHMMSSS) 109 "e20[1-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 110 // Creation Date/Time (ex: cYYYYMMDDHHMMSSS) 111 "c20[1-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d\\d" + 112 // NetCDF 4 suffix 113 ".nc"; 114 115 // This regular expression matches a Suomi NPP Data Product as defined by the 116 // NASA spec in TBD (XXX TJJ - find out where is this documented?) 117 public static final String SUOMI_NPP_REGEX_NASA = 118 // Product Id, Single (ex: VL1BM) 119 // NOTE: These are the only supported NASA data at this time 120 "(VL1BD|VL1BI|VL1BM)" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 121 // Platform - always Suomi NPP 122 "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 123 // Data Start Date (ex: dYYYYMMDD) 124 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 125 // Data Start Time (ex: tHHMM) 126 "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 127 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 128 "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" + 129 // NetCDF 4 suffix 130 ".nc"; 131 132 // This regular expression matches a Suomi NPP geolocation file as defined by the 133 // NASA spec in TBD 134 public static final String SUOMI_GEO_REGEX_NASA = 135 // Product Id, Single (ex: VGEOM) 136 // NOTE: This MUST match the list of product ids in static array in this file! 137 "(VGEOD|VGEOI|VGEOM)" + 138 JPSSUtilities.JPSS_FIELD_SEPARATOR + 139 // Platform - always Suomi NPP 140 "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 141 // Data Start Date (ex: dYYYYMMDD) 142 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 143 // Data Start Time (ex: tHHMM) 144 "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 145 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 146 "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" + 147 // NetCDF 4 suffix 148 ".nc"; 149 150 public static String[] validNASAVariableNames = { 151 "M01", 152 "M02", 153 "M03", 154 "M04", 155 "M05", 156 "M06", 157 "M07", 158 "M08", 159 "M09", 160 "M10", 161 "M11", 162 "M12", 163 "M13", 164 "M14", 165 "M15", 166 "M16", 167 "I01", 168 "I02", 169 "I03", 170 "I04", 171 "I05", 172 "DNB_observations" 173 }; 174 175 public static float[] ATMSChannelCenterFrequencies = { 176 23.8f, 177 31.4f, 178 50.3f, 179 51.76f, 180 52.8f, 181 53.596f, 182 54.40f, 183 54.94f, 184 55.50f, 185 57.29032f, 186 57.29033f, 187 57.29034f, 188 57.29035f, 189 57.29036f, 190 57.29037f, 191 88.20f, 192 165.5f, 193 183.3101f, 194 183.3102f, 195 183.3103f, 196 183.3104f, 197 183.3105f 198 }; 199 200 // the list of valid geolocation product ids 201 public static String[] geoProductIDs = { 202 "GATMO", 203 "GCRSO", 204 "GAERO", 205 "GCLDO", 206 "GDNBO", 207 "GNCCO", 208 "GIGTO", 209 "GIMGO", 210 "GITCO", 211 "GMGTO", 212 "GMODO", 213 "GMTCO", 214 "GNHFO", 215 "GOTCO", 216 "GOSCO", 217 "GONPO", 218 "GONCO", 219 "GCRIO", 220 "GATRO", 221 "IVMIM", 222 "VGEOD", 223 "VGEOI", 224 "VGEOM", 225 "VMUGE" 226 }; 227 228 // This regular expression matches a Suomi NPP geolocation granule, see 229 // NOAA spec in CDFCB-X Volume 1, Page 21 230 public static final String SUOMI_GEO_REGEX_NOAA = 231 // Geo Id, Single (ex: GMODO) 232 // NOTE: This MUST match the list of product ids in static array above! 233 "(GATMO|GCRSO|GAERO|GCLDO|GDNBO|GNCCO|GIGTO|GIMGO|GITCO|" + 234 "GMGTO|GMODO|GMTCO|GNHFO|GOTCO|GOSCO|GONPO|GONCO|GCRIO|GATRO|IVMIM|VMUGE)" + 235 JPSSUtilities.JPSS_FIELD_SEPARATOR + 236 // Spacecraft Id (ex: npp) 237 "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 238 // Data Start Date (ex: dYYYYMMDD) 239 "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 240 // Data Start Time (ex: tHHMMSSS) 241 "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 242 // Data Stop Time (ex: eHHMMSSS) 243 "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 244 // Orbit Number (ex: b00015) 245 "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 246 // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS) 247 "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 248 // Origin (ex: navo) 249 "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR + 250 // Domain (ex: ops) 251 "\\w\\w\\w" + 252 // HDF5 suffix 253 ".h5"; 254 255 /** 256 * Determine if the input variable name is a valid NASA product 257 * 258 * @param varName Variable name to validate. 259 * 260 * @return {@code true} if {@code varName} passes checks. 261 */ 262 263 public static boolean isValidNASA(String varName) { 264 boolean isValid = false; 265 if (varName == null) return isValid; 266 for (String s : validNASAVariableNames) { 267 if (s.equals(varName)) { 268 isValid = true; 269 break; 270 } 271 } 272 return isValid; 273 } 274 275 /** 276 * Determine if the set if filenames constitutes contiguous SNPP granules 277 * of the same geographic coverage. 278 * 279 * @param fileList List of files to validate. 280 * 281 * @return {@code true} if {@code fileList} passes checks. 282 */ 283 284 public static boolean isValidSet(List fileList) { 285 286 // map with filename from start date through orbit will be used for comparisons 287 Map<String, List<String>> metadataMap = new HashMap<String, List<String>>(); 288 289 // Pass 1, populate the list of products selected, and empty maps 290 for (Object o : fileList) { 291 String filename = (String) o; 292 // start at last path separator to clip off absolute paths 293 int lastSeparator = filename.lastIndexOf(File.separatorChar); 294 // NOAA style products 295 if (filename.endsWith(".h5")) { 296 // products go to first underscore, see regex above for more detail 297 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 298 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 299 if (! metadataMap.containsKey(prodStr)) { 300 List<String> l = new ArrayList<String>(); 301 metadataMap.put(prodStr, l); 302 } 303 } 304 // NASA style products 305 if (filename.endsWith(".nc")) { 306 // products end at first underscore, see regex above for more detail 307 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 308 int secondUnderscore = filename.indexOf("_", firstUnderscore + 1); 309 String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore); 310 if (! metadataMap.containsKey(prodStr)) { 311 List<String> l = new ArrayList<String>(); 312 metadataMap.put(prodStr, l); 313 } 314 } 315 } 316 317 // Pass 2, build up the lists of meta data strings and full filenames 318 for (Object o : fileList) { 319 String filename = (String) o; 320 // start at last path separator to clip off absolute paths 321 int lastSeparator = filename.lastIndexOf(File.separatorChar); 322 323 // NOAA style products 324 if (filename.endsWith(".h5")) { 325 // products go to first underscore, see regex above for more detail 326 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 327 // this is the key for the maps 328 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 329 // this is the value for the meta data map - start time through orbit 330 String metaStr = filename.substring(firstUnderscore + 1, firstUnderscore + 39); 331 // get the appropriate list, add the new value 332 List<String> l = (List<String>) metadataMap.get(prodStr); 333 l.add(metaStr); 334 metadataMap.put(prodStr, l); 335 } 336 337 // NASA style products 338 if (filename.endsWith(".nc")) { 339 // products end at first underscore, see regex above for more detail 340 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 341 int secondUnderscore = filename.indexOf("_", firstUnderscore + 1); 342 // this is the key for the maps 343 String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore); 344 // this is the value for the meta data map - date and time 345 int dateIndex = filename.indexOf("_d"); 346 int creationIndex = filename.indexOf("_c"); 347 String metaStr = filename.substring(dateIndex + 1, creationIndex); 348 // get the appropriate list, add the new value 349 List<String> l = (List<String>) metadataMap.get(prodStr); 350 l.add(metaStr); 351 metadataMap.put(prodStr, l); 352 } 353 } 354 355 // loop over metadata map, every list much match the one for ALL other products 356 Set<String> s = metadataMap.keySet(); 357 Iterator iterator = s.iterator(); 358 List prvList = null; 359 while (iterator.hasNext()) { 360 String key = (String) iterator.next(); 361 List l = (List) metadataMap.get(key); 362 for (int i = 0; i < l.size(); i++) { 363 if (prvList != null) { 364 if (! l.equals(prvList)) return false; 365 } 366 } 367 prvList = l; 368 } 369 370 return true; 371 } 372 373 /** 374 * Replace last substring within input string which matches the input with the 375 * provided replacement string. 376 * 377 * @param string 378 * @param substring 379 * @param replacestr 380 * @return 381 */ 382 383 public static String replaceLast(String string, String substring, String replacestr) { 384 // Sanity check on input 385 if (string == null) return null; 386 if ((substring == null) || (replacestr == null)) return string; 387 388 int index = string.lastIndexOf(substring); 389 390 // substring not present 391 if (index == -1) 392 return string; 393 // it's there, swap it 394 return string.substring(0, index) + replacestr 395 + string.substring(index + substring.length()); 396 } 397 398 /** 399 * Determine if a set if filenames which constitutes contiguous SNPP 400 * granules of various products all share the same geolocation data type. 401 * 402 * @param fileList List of files to validate. 403 * @param directory Used when {@literal "GEO"} is not embedded within one 404 * of {@code fileList}. 405 * 406 * @return {@code true} if {@code fileList} passes checks. 407 */ 408 409 public static boolean hasCommonGeo(List fileList, File directory) { 410 Set<String> s = new HashSet<String>(); 411 boolean isCombinedProduct = false; 412 413 // loop through all filenames provided 414 for (Object o : fileList) { 415 isCombinedProduct = false; 416 String filename = (String) o; 417 418 // check the case where GEO is embedded in the data granules 419 int lastSeparator = filename.lastIndexOf(File.separatorChar); 420 int firstUnderscore = filename.indexOf("_", lastSeparator + 1); 421 String prodStr = filename.substring(lastSeparator + 1, firstUnderscore); 422 StringTokenizer st = new StringTokenizer(prodStr, "-"); 423 while (st.hasMoreTokens()) { 424 String singleProd = st.nextToken(); 425 for (int i = 0; i < JPSSUtilities.geoProductIDs.length; i++) { 426 if (singleProd.equals(JPSSUtilities.geoProductIDs[i])) { 427 s.add(singleProd); 428 isCombinedProduct = true; 429 break; 430 } 431 } 432 } 433 // GEO not embedded in file, need to see which GEO file is 434 // referenced in the global attribute 435 if (! isCombinedProduct) { 436 try { 437 String fileFullPath = directory.getAbsolutePath() + File.separator + filename; 438 NetcdfFile ncfile = NetcdfFile.open(fileFullPath); 439 Attribute a = ncfile.findGlobalAttribute("N_GEO_Ref"); 440 if (a != null) { 441 String geoFromAttr = a.getStringValue().substring(0, 5); 442 s.add(geoFromAttr); 443 } 444 ncfile.close(); 445 } catch (IOException ioe) { 446 logger.error("problem reading from attribute", ioe); 447 } 448 } 449 } 450 451 // if the products chosen utilize multiple GEO types, fail the selection 452 if (s.size() > 1) return false; 453 return true; 454 } 455 456}