Source code

001/*
002 * This file is part of McIDAS-V
003 *
004 * Copyright 2007-2024
005 * Space Science and Engineering Center (SSEC)
006 * University of Wisconsin - Madison
007 * 1225 W. Dayton Street, Madison, WI 53706, USA
008 * https://www.ssec.wisc.edu/mcidas/
009 * 
010 * All Rights Reserved
011 * 
012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
013 * some McIDAS-V source code is based on IDV and VisAD source code.  
014 * 
015 * McIDAS-V is free software; you can redistribute it and/or modify
016 * it under the terms of the GNU Lesser Public License as published by
017 * the Free Software Foundation; either version 3 of the License, or
018 * (at your option) any later version.
019 * 
020 * McIDAS-V is distributed in the hope that it will be useful,
021 * but WITHOUT ANY WARRANTY; without even the implied warranty of
022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
023 * GNU Lesser Public License for more details.
024 * 
025 * You should have received a copy of the GNU Lesser Public License
026 * along with this program.  If not, see https://www.gnu.org/licenses/.
027 */
028
029package edu.wisc.ssec.mcidasv.data;
030
031import java.io.IOException;
032import java.nio.file.Paths;
033import java.util.ArrayList;
034import java.util.Arrays;
035import java.util.HashMap;
036import java.util.List;
037import java.util.ListIterator;
038import java.util.Map;
039import java.util.regex.Pattern;
040
041import edu.wisc.ssec.mcidasv.McIDASV;
042
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import ucar.unidata.io.RandomAccessFile;
047import ucar.ma2.Array;
048import ucar.ma2.DataType;
049import ucar.ma2.InvalidRangeException;
050import ucar.ma2.Section;
051import ucar.nc2.Attribute;
052import ucar.nc2.Dimension;
053import ucar.nc2.Group;
054import ucar.nc2.NetcdfFile;
055import ucar.nc2.Variable;
056import ucar.nc2.constants.AxisType;
057import ucar.nc2.constants._Coordinate;
058import ucar.nc2.iosp.AbstractIOServiceProvider;
059import ucar.nc2.util.CancelTask;
060
061import javax.swing.JOptionPane;
062
063/**
064 * @author tommyj
065 *
066 */
067
068public class TropomiIOSP extends AbstractIOServiceProvider {
069
070    private static final String LAT = "latitude";
071
072    private static final String LON = "longitude";
073
074    private static final Logger logger = LoggerFactory.getLogger(TropomiIOSP.class);
075    
076    private static final String BASE_GROUP = "PRODUCT";
077
078    private static final String TROPOMI_FIELD_SEPARATOR = "_";
079
080    // This regular expression matches TROPOMI L2 products
081    private static final String TROPOMI_L2_REGEX =
082            // Mission Name (ex: S5P)
083            "\\w\\w\\w" + TROPOMI_FIELD_SEPARATOR +
084            // Type of data: Real-Time, Offline, Reprocessed, or Products Algorithm Laboratory
085            "(NRTI|OFFL|RPRO|PAL_)" + TROPOMI_FIELD_SEPARATOR +
086            // Product Identifier
087            "(L2_|L1B)" + TROPOMI_FIELD_SEPARATOR +
088            // Product (can be up to six characters, separator-padded if less, e.g. CH4___)
089            "\\w\\w\\w\\w\\w\\w" + TROPOMI_FIELD_SEPARATOR +
090            // Start Date and Time (ex: YYYYmmddTHHMMSS)
091            "20[0-3]\\d[0-1]\\d[0-3]\\dT[0-2]\\d[0-5]\\d[0-6]\\d" + TROPOMI_FIELD_SEPARATOR +
092            // End Date and Time (ex: YYYYmmddTHHMMSS)
093            "20[0-3]\\d[0-1]\\d[0-3]\\dT[0-2]\\d[0-5]\\d[0-6]\\d" + TROPOMI_FIELD_SEPARATOR +
094            // Orbit Number
095            "\\d\\d\\d\\d\\d" + TROPOMI_FIELD_SEPARATOR +
096            // Collection Number
097            "\\d\\d" + TROPOMI_FIELD_SEPARATOR +
098            // Processor Version Number : MMmmpp (Major - Minor - Patch)
099            "\\d\\d\\d\\d\\d\\d" + TROPOMI_FIELD_SEPARATOR +
100            // Creation Date and Time (ex: YYYYmmddTHHMMSS)
101            "20[0-3]\\d[0-1]\\d[0-3]\\dT[0-2]\\d[0-5]\\d[0-6]\\d" +
102            // NetCDF suffix
103            ".nc";
104
105    /** Compiled representation of {@link #TROPOMI_L2_REGEX}. */
106    public static final Pattern TROPOMI_MATCHER =
107        Pattern.compile(TROPOMI_L2_REGEX);
108    
109    /**
110     * Sometimes {@link #isValidFile(RandomAccessFile)} will need to check
111     * Windows paths that look something like {@code /Z:/Users/bob/foo.txt}.
112     * 
113     * <p>This regular expression is used by {@code isValidFile(...)} to
114     * identity these sorts of paths and fix them. Otherwise we'll generate
115     * an {@link java.nio.file.InvalidPathException}.</p>
116     */
117    private static final Pattern BAD_WIN_PATH =
118        Pattern.compile("^/[A-Za-z]:/.+$");
119
120    private static HashMap<String, String> groupMap = new HashMap<String, String>();
121
122    // Dimensions of a product we can work with, init this early
123    private static int[] dimLen = null;
124    
125    private NetcdfFile hdfFile;
126    private static String filename;
127
128    @Override public boolean isValidFile(RandomAccessFile raf) {
129        // Uses the regex defined near top
130        String filePath = raf.getLocation();
131        // TJJ 2022 - For URLs, just fail the match
132        if (filePath.startsWith("https:")) {
133            return false;
134        }
135        if (McIDASV.isWindows() && BAD_WIN_PATH.matcher(filePath).matches()) {
136            filePath = filePath.substring(1);
137        }
138        logger.trace("original path: '{}', path used: '{}'", raf, filePath);
139        filename = Paths.get(filePath).getFileName().toString();
140        return TROPOMI_MATCHER.matcher(filename).matches();
141    }
142
143    @Override public void open(RandomAccessFile raf, NetcdfFile ncfile,
144                     CancelTask cancelTask) throws IOException
145    {
146        logger.trace("TropOMI IOSP open()...");
147
148        // TJJ - kick out anything not supported (most) L2 right now
149        if (filename.contains("_L1B_") || filename.contains("_L2__NP")) {
150            JOptionPane.showMessageDialog(null, "McIDAS-V is unable to read your file. " +
151                    "Only TROPOMI Level 2 Products are supported at this time.", "Warning", JOptionPane.OK_OPTION);
152            return;
153        }
154
155        try {
156            
157            hdfFile = NetcdfFile.open(
158               raf.getLocation(), "ucar.nc2.iosp.hdf5.H5iosp", -1, (CancelTask) null, (Object) null
159            );
160            
161            // Get the dimension lengths for product data if we haven't yet
162            dimLen = getDataShape(hdfFile);
163            // Just logging the dimensions here for debugging purposes
164            for (int i = 0; i < dimLen.length; i++) {
165                logger.trace("Product dimension[" + i + "]: " + dimLen[i]);
166            }
167
168            // Populate map pairing group name to any products we deem suitable for visualization
169            Map<String, List<Variable>> newGroups = getDataVars(hdfFile, dimLen);
170
171            ncfile.addDimension(null, new Dimension("line", dimLen[1]));
172            ncfile.addDimension(null, new Dimension("ele", dimLen[2]));
173            populateDataTree(ncfile, newGroups);
174
175            ncfile.finish();
176        } catch (ClassNotFoundException e) {
177            logger.error("error loading HDF5 IOSP", e);
178        } catch (IllegalAccessException e) {
179            logger.error("java reflection error", e);
180        } catch (InstantiationException e) {
181            logger.error("error instantiating", e);
182        }
183    }
184
185    /*
186     * Loop over all data looking for the products we can display
187     */
188
189    private static Map<String, List<Variable>> getDataVars(NetcdfFile hdf, int[] dataShape) {
190        List<Variable> variables = hdf.getVariables();
191        Map<String, List<Variable>> groupsToDataVars = new HashMap<>(variables.size());
192        for (Variable v : variables) {
193            if (Arrays.equals(dataShape, v.getShape())) {
194                String groupName = v.getGroup().getFullNameEscaped();
195                if (! groupsToDataVars.containsKey(groupName)) {
196                    groupsToDataVars.put(groupName, new ArrayList<Variable>(variables.size()));
197                }
198                groupsToDataVars.get(groupName).add(v);
199            }
200        }
201        return groupsToDataVars;
202    }
203
204    /*
205     * Create the group structure and data products for our McV output
206     */
207
208    private static void populateDataTree(NetcdfFile ncOut, Map<String, List<Variable>> groupsToVars)
209    {
210        for (Map.Entry<String, List<Variable>> e : groupsToVars.entrySet()) {
211            Group g = new Group(ncOut, null, e.getKey());
212
213            logger.trace("Adding Group: " + g.getFullName());
214            // Newly created groups will have path separators converted to underscores
215            // We'll need to map back to the original group name for file access
216            groupMap.put(g.getFullName(), e.getKey());
217
218            ncOut.addGroup(null, g);
219
220            for (Variable v : e.getValue()) {
221                logger.trace("Adding Variable: " + v.getFullNameEscaped());
222
223                // TJJ Aug 2020
224                // Operational change described in
225                // https://mcidas.ssec.wisc.edu/inquiry-v/?inquiry=2918
226                // This caused invalid units of "milliseconds since ..." in delta_time attribute
227                // to prevent variables from Product group to load
228                if (v.getShortName().equals("delta_time")) {
229                    for (Attribute attribute : v.getAttributes()) {
230                        if (attribute.getShortName().equals("units")) {
231                            if (attribute.getStringValue().startsWith("milliseconds since")) {
232                                logger.warn("Altering invalid units attribute value");
233                                v.addAttribute(new Attribute("units", "milliseconds"));
234                            }
235                        }
236                    }
237                }
238
239                addVar(ncOut, g, v);
240            }
241
242        }
243    }
244
245    /**
246     * Fulfill data requests
247     * @return Array - an array with the requested data subset
248     */
249
250    @Override public Array readData(Variable variable, Section section)
251        throws IOException, InvalidRangeException
252    {
253        String variableName = variable.getShortName();
254        
255        String groupName = groupMap.get(variable.getGroup().getFullName());
256        logger.trace("looking for Group: " + groupName);
257        Group hdfGroup = hdfFile.findGroup(groupName);
258        Array result;
259
260        logger.trace("TropOMI IOSP readData(), var name: " + variableName);
261        Variable hdfVariable = hdfGroup.findVariable(variableName);
262        
263        logger.trace("found var: " + hdfVariable.getFullName() + 
264                " in group: " + hdfVariable.getGroup().getFullName());
265        // Need to knock off 1st dimension for Lat and Lon too...
266        int[] origin = { 0, 0, 0 };
267        int[] size = { 1, dimLen[1], dimLen[2] };
268        logger.trace("reading size: 1, " + dimLen[1] + ", " + dimLen[2]);
269        result = hdfVariable.read(origin, size).reduce();
270        return result;
271    }
272
273    /*
274     * Test whether file in question is a valid product for this IOSP
275     * This method MUST BE LIGHTNING FAST, since it's part of the system
276     * process of attempting to infer the proper handler when the user
277     * is not certain what the best way to handle the data might be.
278     */
279
280    private static boolean validProduct(Variable variable) {
281        int[] varShape = variable.getShape();
282        if (varShape.length != dimLen.length) return false;
283        // Same dimensions, make sure each individual dimension matches
284        for (int i = 0; i < varShape.length; i++) {
285            if (varShape[i] != dimLen[i]) return false;
286        }
287        return true;
288    }
289
290    /*
291     * Get the shape of valid data products. We consider anything that matches
292     * the geolocation bounds to be valid.
293     */
294
295    private static int[] getDataShape(NetcdfFile hdf) {
296        Group productGroup = hdf.findGroup(BASE_GROUP);
297        // Shape of valid data products will match that of geolocation, so base on LAT or LON
298        Variable geoVar = productGroup.findVariable(LAT);
299        return new int[] {
300            geoVar.getDimension(0).getLength(),
301            geoVar.getDimension(1).getLength(),
302            geoVar.getDimension(2).getLength()
303        };
304    }
305
306    /*
307     * Add a variable to the set of available products.
308     */
309
310    private static void addVar(NetcdfFile nc, Group g, Variable vIn) {
311
312        logger.trace("Evaluating: " + vIn.getFullName());
313        if (validProduct(vIn)) {
314            Variable v = new Variable(nc, g, null, vIn.getShortName(), DataType.FLOAT, "line ele");
315            if (vIn.getShortName().equals(LAT)) {
316                v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));
317                v.addAttribute(new Attribute("coordinates", "latitude longitude"));
318                logger.trace("including: " + vIn.getFullName());
319            } else if (vIn.getShortName().equals(LON)) {
320                v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));
321                v.addAttribute(new Attribute("coordinates", "latitude longitude"));
322                logger.trace("including: " + vIn.getFullName());
323            } else {
324                v.addAttribute(new Attribute("coordinates", "latitude longitude"));
325                logger.trace("including: " + vIn.getFullName());
326            }
327            List<Attribute> attList = vIn.getAttributes();
328            for (Attribute a : attList) {
329                v.addAttribute(a);
330            }
331            logger.trace("adding vname: " + vIn.getFullName() + " to group: " + g.getFullName());
332            
333            g.addVariable(v);
334        }
335    }
336    
337    @Override public String getFileTypeId() {
338        return "TropOMI";
339    }
340
341    @Override public String getFileTypeDescription() {
342        return "TROPOspheric Monitoring Instrument";
343    }
344
345    @Override public void close() throws IOException {
346        hdfFile.close();
347    }
348
349    public static void main(String args[]) throws IOException, IllegalAccessException, InstantiationException {
350        NetcdfFile.registerIOProvider(TropomiIOSP.class);
351    }
352    
353}