001/*
002 * This file is part of McIDAS-V
003 *
004 * Copyright 2007-2018
005 * Space Science and Engineering Center (SSEC)
006 * University of Wisconsin - Madison
007 * 1225 W. Dayton Street, Madison, WI 53706, USA
008 * https://www.ssec.wisc.edu/mcidas
009 * 
010 * All Rights Reserved
011 * 
012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
013 * some McIDAS-V source code is based on IDV and VisAD source code.  
014 * 
015 * McIDAS-V is free software; you can redistribute it and/or modify
016 * it under the terms of the GNU Lesser Public License as published by
017 * the Free Software Foundation; either version 3 of the License, or
018 * (at your option) any later version.
019 * 
020 * McIDAS-V is distributed in the hope that it will be useful,
021 * but WITHOUT ANY WARRANTY; without even the implied warranty of
022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
023 * GNU Lesser Public License for more details.
024 * 
025 * You should have received a copy of the GNU Lesser Public License
026 * along with this program.  If not, see http://www.gnu.org/licenses.
027 */
028
029package edu.wisc.ssec.mcidasv.data.hydra;
030
031import java.io.File;
032import java.io.IOException;
033import java.util.ArrayList;
034import java.util.HashMap;
035import java.util.HashSet;
036import java.util.Iterator;
037import java.util.List;
038import java.util.Map;
039import java.util.Set;
040import java.util.StringTokenizer;
041
042import ucar.nc2.Attribute;
043import ucar.nc2.NetcdfFile;
044
045/**
046 * Utility class to support Joint Polar Satellite System (JPSS) functionality.
047 * Documentation referenced is from Suomi NPP Common Data Format Control Book.
048 * See:
049 * http://npp.gsfc.nasa.gov/documents.html
050 * 
051 * @author tommyj
052 *
053 */
054
055public abstract class JPSSUtilities {
056   
057        public static final String JPSS_FIELD_SEPARATOR = "_";
058        public static final int NASA_CREATION_DATE_INDEX = 28;
059        public static final int NOAA_CREATION_DATE_INDEX = 35;
060        
061        // Flags to check for SIPS data V2.0.0 and higher
062        // Augments single fill value from prior versions
063        public static final String SIPS_BOWTIE_DELETED_FLAG = "Bowtie_Deleted";
064        public static final String SIPS_FLAG_MEANINGS_ATTRIBUTE = "flag_meanings";
065        public static final String SIPS_FLAG_VALUES_ATTRIBUTE = "flag_values";
066        
067        // This regular expression matches a Suomi NPP Data Product as defined by the 
068        // NOAA spec in CDFCB-X Volume 1, Page 21
069        public static final String SUOMI_NPP_REGEX_NOAA =
070                // Product Id, Multiple (ex: VSSTO-GATMO-VSLTO)
071                "(\\w\\w\\w\\w\\w-)*" + 
072                // Product Id, Single (ex: VSSTO)
073                "\\w\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
074                // Spacecraft Id (ex: npp)
075                "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
076                // Data Start Date (ex: dYYYYMMDD)
077                "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
078                // Data Start Time (ex: tHHMMSSS)
079                "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
080                // Data Stop Time (ex: eHHMMSSS)
081                "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
082                // Orbit Number (ex: b00015)
083                "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
084                // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS)
085                "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
086                // Origin (ex: navo)
087                "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
088                // Domain (ex: ops)
089                "\\w\\w\\w" + 
090                // HDF5 suffix
091                ".h5";
092        
093        // This regular expression matches a Suomi NPP Data Product as defined by the 
094        // NASA spec in TBD (XXX TJJ - find out where is this documented?)
095        public static final String SUOMI_NPP_REGEX_NASA =
096                        // Product Id, Single (ex: VL1BM)
097                        // NOTE: These are the only supported NASA data at this time
098                "(VL1BD|VL1BI|VL1BM)" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
099                // Platform - always Suomi NPP
100                "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
101                // Data Start Date (ex: dYYYYMMDD)
102                "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
103                // Data Start Time (ex: tHHMM)
104                "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
105                // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS)
106                "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" +
107                // NetCDF 4 suffix
108                ".nc";
109        
110        // This regular expression matches a Suomi NPP geolocation file as defined by the 
111        // NASA spec in TBD
112        public static final String SUOMI_GEO_REGEX_NASA =
113                        // Product Id, Single (ex: VGEOM)
114                        // NOTE: This MUST match the list of product ids in static array in this file!
115                "(VGEOD|VGEOI|VGEOM)" + 
116                        JPSSUtilities.JPSS_FIELD_SEPARATOR +
117                // Platform - always Suomi NPP
118                "snpp" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
119                // Data Start Date (ex: dYYYYMMDD)
120                "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
121                // Data Start Time (ex: tHHMM)
122                "t[0-2]\\d[0-5]\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
123                // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS)
124                "c20[0-3]\\d[0-1]\\d[0-3]\\d\\d\\d\\d\\d\\d\\d" +
125                // NetCDF 4 suffix
126                ".nc";
127        
128        public static String[] validNASAVariableNames = {
129                "M01",
130                "M02",
131                "M03",
132                "M04",
133                "M05",
134                "M06",
135                "M07",
136                "M08",
137                "M09",
138                "M10",
139                "M11",
140                "M12",
141                "M13",
142                "M14",
143                "M15",
144                "M16",
145                "I01",
146                "I02",
147                "I03",
148                "I04",
149                "I05",
150                "DNB_observations"
151        };
152        
153        public static float[] ATMSChannelCenterFrequencies = {
154                23.8f,
155                31.4f,
156                50.3f,
157                51.76f,
158                52.8f,
159                53.596f,
160                54.40f,
161                54.94f,
162                55.50f,
163                57.29032f,
164                57.29033f,
165                57.29034f,
166                57.29035f,
167                57.29036f,
168                57.29037f,
169                88.20f,
170                165.5f,
171                183.3101f,
172                183.3102f,
173                183.3103f,
174                183.3104f,
175                183.3105f
176        };
177        
178        // the list of valid geolocation product ids
179        public static String[] geoProductIDs = {
180        "GATMO",
181        "GCRSO",
182        "GAERO",
183        "GCLDO",
184        "GDNBO",
185        "GNCCO",
186        "GIGTO",
187        "GIMGO",
188        "GITCO",
189        "GMGTO",
190        "GMODO",
191        "GMTCO",
192        "GNHFO",
193        "GOTCO",
194        "GOSCO",
195        "GONPO",
196        "GONCO",
197        "GCRIO",
198        "GATRO",
199        "IVMIM",
200        "VGEOD",
201        "VGEOI",
202        "VGEOM",
203        "VMUGE"
204        };  
205        
206        // This regular expression matches a Suomi NPP geolocation granule, see 
207        // NOAA spec in CDFCB-X Volume 1, Page 21
208        public static final String SUOMI_GEO_REGEX_NOAA =
209                // Geo Id, Single (ex: GMODO)
210                        // NOTE: This MUST match the list of product ids in static array above!
211                "(GATMO|GCRSO|GAERO|GCLDO|GDNBO|GNCCO|GIGTO|GIMGO|GITCO|" + 
212                        "GMGTO|GMODO|GMTCO|GNHFO|GOTCO|GOSCO|GONPO|GONCO|GCRIO|GATRO|IVMIM|VMUGE)" + 
213                        JPSSUtilities.JPSS_FIELD_SEPARATOR +
214                // Spacecraft Id (ex: npp)
215                "\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
216                // Data Start Date (ex: dYYYYMMDD)
217                "d20[0-3]\\d[0-1]\\d[0-3]\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
218                // Data Start Time (ex: tHHMMSSS)
219                "t[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
220                // Data Stop Time (ex: eHHMMSSS)
221                "e[0-2]\\d[0-5]\\d[0-6]\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
222                // Orbit Number (ex: b00015)
223                "b\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
224                // Creation Date (ex: cYYYYMMDDHHMMSSSSSSSS)
225                "c20[0-3]\\d[0-1]\\d[0-3]\\d[0-2]\\d[0-5]\\d[0-6]\\d\\d\\d\\d\\d\\d\\d" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
226                // Origin (ex: navo)
227                "\\w\\w\\w\\w" + JPSSUtilities.JPSS_FIELD_SEPARATOR +
228                // Domain (ex: ops)
229                "\\w\\w\\w" + 
230                // HDF5 suffix
231                ".h5";
232        
233        /**
234         * Determine if the input variable name is a valid NASA product
235         *
236         * @param varName Variable name to validate.
237         *
238         * @return {@code true} if {@code varName} passes checks.
239         */
240        
241        public static boolean isValidNASA(String varName) {
242                boolean isValid = false;
243                if (varName == null) return isValid; 
244                for (String s : validNASAVariableNames) {
245                        if (s.equals(varName)) {
246                                isValid = true;
247                                break;
248                        }
249                }
250                return isValid;
251        }
252        
253        /**
254         * Determine if the set if filenames constitutes contiguous SNPP granules
255         * of the same geographic coverage.
256         *
257         * @param fileList List of files to validate.
258         *
259         * @return {@code true} if {@code fileList} passes checks.
260         */
261        
262        public static boolean isValidSet(List fileList) {
263                
264                // map with filename from start date through orbit will be used for comparisons
265        Map<String, List<String>> metadataMap = new HashMap<String, List<String>>();
266        
267        // Pass 1, populate the list of products selected, and empty maps
268        for (Object o : fileList) {
269                String filename = (String) o;
270                // start at last path separator to clip off absolute paths
271                int lastSeparator = filename.lastIndexOf(File.separatorChar);
272                // NOAA style products
273                if (filename.endsWith(".h5")) {
274                        // products go to first underscore, see regex above for more detail
275                        int firstUnderscore = filename.indexOf("_", lastSeparator + 1);
276                        String prodStr = filename.substring(lastSeparator + 1, firstUnderscore);
277                        if (! metadataMap.containsKey(prodStr)) {
278                                        List<String> l = new ArrayList<String>();
279                                        metadataMap.put(prodStr, l);
280                        }
281                }
282                // NASA style products
283                if (filename.endsWith(".nc")) {
284                        // products end at first underscore, see regex above for more detail
285                        int firstUnderscore = filename.indexOf("_", lastSeparator + 1);
286                        int secondUnderscore = filename.indexOf("_", firstUnderscore + 1);
287                        String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore);
288                        if (! metadataMap.containsKey(prodStr)) {
289                                        List<String> l = new ArrayList<String>();
290                                        metadataMap.put(prodStr, l);
291                        }
292                }
293        }
294        
295        // Pass 2, build up the lists of meta data strings and full filenames
296        for (Object o : fileList) {
297                String filename = (String) o;
298                // start at last path separator to clip off absolute paths
299                int lastSeparator = filename.lastIndexOf(File.separatorChar);
300                
301                // NOAA style products
302                if (filename.endsWith(".h5")) {
303                        // products go to first underscore, see regex above for more detail
304                        int firstUnderscore = filename.indexOf("_", lastSeparator + 1);
305                        // this is the key for the maps
306                        String prodStr = filename.substring(lastSeparator + 1, firstUnderscore);
307                        // this is the value for the meta data map - start time through orbit 
308                        String metaStr = filename.substring(firstUnderscore + 1, firstUnderscore + 39);
309                        // get the appropriate list, add the new value
310                        List<String> l = (List<String>) metadataMap.get(prodStr);
311                        l.add(metaStr);
312                        metadataMap.put(prodStr, l);
313                }
314                
315                // NASA style products
316                if (filename.endsWith(".nc")) {
317                        // products end at first underscore, see regex above for more detail
318                        int firstUnderscore = filename.indexOf("_", lastSeparator + 1);
319                        int secondUnderscore = filename.indexOf("_", firstUnderscore + 1);
320                        // this is the key for the maps
321                        String prodStr = filename.substring(firstUnderscore + 1, secondUnderscore);
322                        // this is the value for the meta data map - date and time 
323                        int dateIndex = filename.indexOf("_d");
324                        int creationIndex = filename.indexOf("_c");
325                        String metaStr = filename.substring(dateIndex + 1, creationIndex);
326                        // get the appropriate list, add the new value
327                        List<String> l = (List<String>) metadataMap.get(prodStr);
328                        l.add(metaStr);
329                        metadataMap.put(prodStr, l);
330                }
331        }
332        
333        // loop over metadata map, every list much match the one for ALL other products
334        Set<String> s = metadataMap.keySet();
335        Iterator iterator = s.iterator();
336        List prvList = null;
337        while (iterator.hasNext()) {
338                String key = (String) iterator.next();
339                List l = (List) metadataMap.get(key);
340                for (int i = 0; i < l.size(); i++) {
341                        if (prvList != null) {
342                                if (! l.equals(prvList)) return false;
343                        }
344                }
345                prvList = l;
346        }
347        
348                return true;
349        }
350
351        /**
352         * Replace last substring within input string which matches the input with the 
353         * provided replacement string.
354         * 
355         * @param string
356         * @param substring
357         * @param replacestr
358         * @return
359         */
360        
361        public static String replaceLast(String string, String substring, String replacestr) {
362                // Sanity check on input
363                if (string == null) return null;
364                if ((substring == null) || (replacestr == null)) return string;
365                
366                int index = string.lastIndexOf(substring);
367                
368                // substring not present
369                if (index == -1)
370                        return string;
371                // it's there, swap it
372                return string.substring(0, index) + replacestr
373                                + string.substring(index + substring.length());
374        }
375        
376        /**
377         * Determine if a set if filenames which constitutes contiguous SNPP
378         * granules of various products all share the same geolocation data type.
379         *
380         * @param fileList List of files to validate.
381         * @param directory Used when {@literal "GEO"} is not embedded within one
382         *                  of {@code fileList}.
383         *
384         * @return {@code true} if {@code fileList} passes checks.
385         */
386        
387        public static boolean hasCommonGeo(List fileList, File directory) {
388                Set<String> s = new HashSet<String>();
389                boolean isCombinedProduct = false;
390                
391                // loop through all filenames provided
392        for (Object o : fileList) {
393                isCombinedProduct = false;
394                String filename = (String) o;
395                
396                // check the case where GEO is embedded in the data granules
397                int lastSeparator = filename.lastIndexOf(File.separatorChar);
398                int firstUnderscore = filename.indexOf("_", lastSeparator + 1);
399                String prodStr = filename.substring(lastSeparator + 1, firstUnderscore);
400            StringTokenizer st = new StringTokenizer(prodStr, "-");
401            while (st.hasMoreTokens()) {
402                String singleProd = st.nextToken();
403                for (int i = 0; i < JPSSUtilities.geoProductIDs.length; i++) {
404                        if (singleProd.equals(JPSSUtilities.geoProductIDs[i])) {
405                                s.add(singleProd);
406                                isCombinedProduct = true;
407                                break;
408                        }
409                }
410            }
411            // GEO not embedded in file, need to see which GEO file is
412                        // referenced in the global attribute
413            if (! isCombinedProduct) {
414                try {
415                        String fileFullPath = directory.getAbsolutePath() + File.separator + filename;
416                                        NetcdfFile ncfile = NetcdfFile.open(fileFullPath);
417                                        Attribute a = ncfile.findGlobalAttribute("N_GEO_Ref");
418                                        if (a != null) {
419                                                String geoFromAttr = a.getStringValue().substring(0, 5);
420                                                s.add(geoFromAttr);
421                                        }
422                                        ncfile.close();
423                                } catch (IOException ioe) {
424                                        ioe.printStackTrace();
425                                }
426            }
427        }
428        
429        // if the products chosen utilize multiple GEO types, fail the selection
430        if (s.size() > 1) return false;
431                return true;
432        }
433        
434}