Source code

001/*
002 * This file is part of McIDAS-V
003 *
004 * Copyright 2007-2024
005 * Space Science and Engineering Center (SSEC)
006 * University of Wisconsin - Madison
007 * 1225 W. Dayton Street, Madison, WI 53706, USA
008 * https://www.ssec.wisc.edu/mcidas/
009 * 
010 * All Rights Reserved
011 * 
012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
013 * some McIDAS-V source code is based on IDV and VisAD source code.  
014 * 
015 * McIDAS-V is free software; you can redistribute it and/or modify
016 * it under the terms of the GNU Lesser Public License as published by
017 * the Free Software Foundation; either version 3 of the License, or
018 * (at your option) any later version.
019 * 
020 * McIDAS-V is distributed in the hope that it will be useful,
021 * but WITHOUT ANY WARRANTY; without even the implied warranty of
022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
023 * GNU Lesser Public License for more details.
024 * 
025 * You should have received a copy of the GNU Lesser Public License
026 * along with this program.  If not, see https://www.gnu.org/licenses/.
027 */
028
029package edu.wisc.ssec.mcidasv.data.hydra;
030
031import java.util.Map;
032import java.util.Objects;
033
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036
037import edu.wisc.ssec.mcidasv.data.QualityFlag;
038import visad.util.Util;
039
040public class RangeProcessor {
041
042    private static final Logger logger =
043        LoggerFactory.getLogger(RangeProcessor.class);
044
045    static RangeProcessor createRangeProcessor(MultiDimensionReader reader,
046            Map<String, Object> metadata) throws Exception {
047        if (reader instanceof GranuleAggregation) {
048            return new AggregationRangeProcessor((GranuleAggregation) reader, metadata);
049        }
050
051        if (metadata.get("scale_name") == null) {
052            String product_name = (String) metadata.get(SwathAdapter.product_name);
053            if (Objects.equals(product_name, "IASI_L1C_xxx")) {
054                return new IASI_RangeProcessor();
055            }
056            return null;
057        } else {
058            String product_name = (String) metadata.get(ProfileAlongTrack.product_name);
059            if (Objects.equals(product_name, "2B-GEOPROF")) {
060                return new CloudSat_2B_GEOPROF_RangeProcessor(reader, metadata);
061            } else {
062                return new RangeProcessor(reader, metadata);
063            }
064        }
065    }
066
067    MultiDimensionReader reader;
068    Map<String, Object> metadata;
069
070    float[] scale = null;
071    float[] offset = null;
072    double[] missing = null;
073    double[] valid_range = null;
074    double valid_low = -Double.MAX_VALUE;
075    double valid_high = Double.MAX_VALUE;
076
077    boolean unpack = false;
078    boolean unsigned = false;
079    boolean rangeCheckBeforeScaling = true;
080
081    int scaleOffsetLen = 1;
082
083    String multiScaleDimName = SpectrumAdapter.channelIndex_name;
084    boolean hasMultiDimensionScale = false;
085
086    int multiScaleDimensionIndex = 0;
087
088    int soIndex = 0;
089
090    public RangeProcessor() {
091    }
092
093    public RangeProcessor(float scale, float offset, float valid_low, float valid_high,
094            float missing) {
095        this.scale = new float[] { scale };
096        this.offset = new float[] { offset };
097        this.missing = new double[] { missing };
098        this.valid_low = valid_low;
099        this.valid_high = valid_high;
100    }
101
102    public RangeProcessor(MultiDimensionReader reader, Map<String, Object> metadata,
103            String multiScaleDimName) throws Exception {
104        this(reader, metadata);
105        this.multiScaleDimName = multiScaleDimName;
106    }
107
108    public RangeProcessor(MultiDimensionReader reader, Map<String, Object> metadata)
109            throws Exception {
110        this.reader = reader;
111        this.metadata = metadata;
112
113        if (metadata.get("unpack") != null) {
114            unpack = true;
115        }
116
117        if (metadata.get("unsigned") != null) {
118            unsigned = true;
119        }
120
121        if (metadata.get("range_check_after_scaling") != null) {
122            String s = (String) metadata.get("range_check_after_scaling");
123            logger.debug("range_check_after_scaling: " + s);
124            rangeCheckBeforeScaling = false;
125        }
126
127        String array_name = (String) metadata.get("array_name");
128
129        scale = getAttributeAsFloatArray(array_name, (String) metadata.get("scale_name"));
130
131        offset = getAttributeAsFloatArray(array_name, (String) metadata.get("offset_name"));
132
133        if (scale != null) {
134            scaleOffsetLen = scale.length;
135
136            if (offset != null) {
137                if (scale.length != offset.length) {
138                    throw new Exception(
139                            "RangeProcessor: scale and offset array lengths must be equal");
140                }
141            } else {
142                offset = new float[scaleOffsetLen];
143                for (int i = 0; i < offset.length; i++)
144                    offset[i] = 0f;
145            }
146
147        }
148
149        missing = getAttributeAsDoubleArray(array_name, (String) metadata.get("fill_value_name"));
150
151        // if we are working with unsigned data, need to convert missing vals to
152        // unsigned too
153        if (unsigned) {
154            if (missing != null) {
155                for (int i = 0; i < missing.length; i++) {
156                    missing[i] = (float) Util.unsignedShortToInt((short) missing[i]);
157                }
158            }
159        }
160
161        String metaStr = (String) metadata.get("valid_range");
162        // attr name not supplied, so try the convention default
163        if (metaStr == null) {
164            metaStr = "valid_range";
165        }
166
167        valid_range = getAttributeAsDoubleArray(array_name, metaStr);
168        if (valid_range != null) {
169
170            valid_low = valid_range[0];
171            valid_high = valid_range[1];
172
173            if (valid_range[0] > valid_range[1]) {
174                valid_low = valid_range[1];
175                valid_high = valid_range[0];
176            }
177        } else {
178            metaStr = (String) metadata.get("valid_low");
179            if (metaStr == null) { // attr name not supplied, so try the
180                                   // convention default
181                metaStr = "valid_min";
182            }
183            double[] dblA = getAttributeAsDoubleArray(array_name, metaStr);
184            if (dblA != null) {
185                valid_low = dblA[0];
186            }
187
188            metaStr = (String) metadata.get("valid_high");
189            if (metaStr == null) { // attr name not supplied, so try the
190                                   // convention default
191                metaStr = "valid_max";
192            }
193            dblA = getAttributeAsDoubleArray(array_name, metaStr);
194            if (dblA != null) {
195                valid_high = dblA[0];
196            }
197        }
198
199        if (rangeCheckBeforeScaling) {
200            if (unsigned) {
201                if (valid_low != -Double.MAX_VALUE) {
202                    valid_low = (double) Util.unsignedShortToInt((short) valid_low);
203                }
204                if (valid_high != Double.MAX_VALUE) {
205                    valid_high = (double) Util.unsignedShortToInt((short) valid_high);
206                }
207            }
208        }
209
210        String str = (String) metadata.get("multiScaleDimensionIndex");
211        hasMultiDimensionScale = (str != null);
212        multiScaleDimensionIndex = (str != null) ? Integer.parseInt(str) : 0;
213    }
214
215    public float[] getAttributeAsFloatArray(String arrayName, String attrName) throws Exception {
216        float[] fltArray = null;
217        HDFArray arrayAttr = reader.getArrayAttribute(arrayName, attrName);
218
219        if (arrayAttr != null) {
220
221            if (arrayAttr.getType().equals(Float.TYPE)) {
222                float[] attr = (float[]) arrayAttr.getArray();
223                fltArray = new float[attr.length];
224                for (int k = 0; k < attr.length; k++)
225                    fltArray[k] = attr[k];
226            } else if (arrayAttr.getType().equals(Short.TYPE)) {
227                short[] attr = (short[]) arrayAttr.getArray();
228                fltArray = new float[attr.length];
229                for (int k = 0; k < attr.length; k++)
230                    fltArray[k] = (float) attr[k];
231            } else if (arrayAttr.getType().equals(Integer.TYPE)) {
232                int[] attr = (int[]) arrayAttr.getArray();
233                fltArray = new float[attr.length];
234                for (int k = 0; k < attr.length; k++)
235                    fltArray[k] = (float) attr[k];
236            } else if (arrayAttr.getType().equals(Double.TYPE)) {
237                double[] attr = (double[]) arrayAttr.getArray();
238                fltArray = new float[attr.length];
239                for (int k = 0; k < attr.length; k++)
240                    fltArray[k] = (float) attr[k];
241            }
242
243        }
244
245        return fltArray;
246    }
247
248    public double[] getAttributeAsDoubleArray(String arrayName, String attrName) throws Exception {
249        if (attrName == null) {
250            return null;
251        }
252
253        double[] dblArray = null;
254        HDFArray arrayAttr = null;
255        try {
256            arrayAttr = reader.getArrayAttribute(arrayName, attrName);
257        } catch (Exception e) {
258            String msg = String.format("Problem getting array attribute. " +
259                "arrayName: '%s', attrName: '%s'", arrayName, attrName);
260            logger.error(msg, e);
261        }
262
263        if (arrayAttr != null) {
264
265            if (arrayAttr.getType().equals(Float.TYPE)) {
266                float[] attr = (float[]) arrayAttr.getArray();
267                dblArray = new double[attr.length];
268                for (int k = 0; k < attr.length; k++)
269                    dblArray[k] = attr[k];
270            } else if (arrayAttr.getType().equals(Short.TYPE)) {
271                short[] attr = (short[]) arrayAttr.getArray();
272                dblArray = new double[attr.length];
273                for (int k = 0; k < attr.length; k++) {
274                    if (unsigned) {
275                        dblArray[k] = (double) Util.unsignedShortToInt((short) attr[k]);
276                    } else {
277                        dblArray[k] = (double) attr[k];
278                    }
279                }
280            } else if (arrayAttr.getType().equals(Integer.TYPE)) {
281                int[] attr = (int[]) arrayAttr.getArray();
282                dblArray = new double[attr.length];
283                for (int k = 0; k < attr.length; k++)
284                    dblArray[k] = (double) attr[k];
285            } else if (arrayAttr.getType().equals(Double.TYPE)) {
286                double[] attr = (double[]) arrayAttr.getArray();
287                dblArray = new double[attr.length];
288                for (int k = 0; k < attr.length; k++)
289                    dblArray[k] = (double) attr[k];
290            } else if (arrayAttr.getType().equals(Byte.TYPE)) {
291                byte[] attr = (byte[]) arrayAttr.getArray();
292                dblArray = new double[attr.length];
293                for (int k = 0; k < attr.length; k++)
294                    dblArray[k] = (double) attr[k];
295            } else if (arrayAttr.getType().equals(String.class)) {
296                String[] attr = (String[]) arrayAttr.getArray();
297                dblArray = new double[attr.length];
298                for (int k = 0; k < attr.length; k++)
299                    dblArray[k] = Double.valueOf(attr[0]);
300            }
301        }
302
303        return dblArray;
304    }
305
306    /**
307     * Process a range of data from an array of {@code byte} values where bytes
308     * are packed bit or multi-bit fields of quality flags. Based on info in a
309     * {@link QualityFlag} object passed in, we extract and return values for
310     * that flag.
311     * 
312     * @param values
313     *            Input byte values. Cannot be {@code null}.
314     * @param subset
315     *            Optional subset.
316     * @param qf
317     *            Quality flag.
318     *
319     * @return Processed range.
320     */
321
322    public float[] processRangeQualityFlag(byte[] values, Map<String, double[]> subset, QualityFlag qf) {
323
324        if (subset != null) {
325            if (subset.get(multiScaleDimName) != null) {
326                soIndex = (int) (subset.get(multiScaleDimName))[0];
327            }
328        }
329
330        float[] newValues = new float[values.length];
331
332        float val = 0f;
333        int bitOffset = qf.getBitOffset();
334        int divisor = -1;
335
336        // map bit offset to a divisor
337        switch (bitOffset) {
338            case 1:
339                divisor = 2;
340                break;
341            case 2:
342                divisor = 4;
343                break;
344            case 3:
345                divisor = 8;
346                break;
347            case 4:
348                divisor = 16;
349                break;
350            case 5:
351                divisor = 32;
352                break;
353            case 6:
354                divisor = 64;
355                break;
356            case 7:
357                divisor = 128;
358                break;
359            default:
360                divisor = 1;
361                break;
362        }
363
364        // now map bit width to a mask
365        int numBits = qf.getNumBits();
366        int mask = -1;
367        switch (numBits) {
368            case 1:
369                mask = (int) 0x00000001;
370                break;
371            case 2:
372                mask = (int) 0x00000003;
373                break;
374            case 3:
375                mask = (int) 0x00000007;
376                break;
377            case 4:
378                mask = (int) 0x0000000F;
379                break;
380            case 5:
381                mask = (int) 0x0000001F;
382                break;
383            case 6:
384                mask = (int) 0x0000003F;
385                break;
386            case 7:
387                mask = (int) 0x0000007F;
388                break;
389            default:
390                mask = (int) 0x00000000;
391                break;
392        }
393
394        int i = 0;
395        for (int k = 0; k < values.length; k++) {
396            val = (float) values[k];
397            i = Util.unsignedByteToInt(values[k]);
398            val = (float) ((i / divisor) & mask);
399            newValues[k] = val;
400        }
401
402        return newValues;
403    }
404
405    /**
406     * Process a range of data from an array of {@code byte} values.
407     * 
408     * @param values
409     *            Input {@code byte} values. Cannot be {@code null}.
410     * @param subset
411     *            Optional subset.
412     *
413     * @return Processed range.
414     */
415    
416    public float[] processRange(byte[] values, Map<String, double[]> subset) {
417
418        int multiScaleDimLen = 1;
419
420        if (subset != null) {
421            if (subset.get(multiScaleDimName) != null) {
422                double[] coords = subset.get(multiScaleDimName);
423                soIndex = (int) coords[0];
424                multiScaleDimLen = (int) (coords[1] - coords[0] + 1.0);
425            }
426        }
427
428        float[] new_values = new float[values.length];
429
430        float val = 0f;
431        int i = 0;
432        boolean isMissing = false;
433
434        for (int k = 0; k < values.length; k++) {
435
436            val = (float) values[k];
437            if (unsigned) {
438                i = Util.unsignedByteToInt(values[k]);
439                val = (float) i;
440            }
441
442            // first, check the (possibly multiple) missing values
443            isMissing = false;
444            if (missing != null) {
445                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
446                    if (val == missing[mvIdx]) {
447                        isMissing = true;
448                        break;
449                    }
450                }
451            }
452
453            if (isMissing) {
454                new_values[k] = Float.NaN;
455                continue;
456            }
457
458            if (rangeCheckBeforeScaling) {
459                if ((val < valid_low) || (val > valid_high)) {
460                    new_values[k] = Float.NaN;
461                    continue;
462                }
463            }
464
465            if (scale != null) {
466                if (unpack) {
467                    if (multiScaleDimLen == 1) {
468                        new_values[k] = (scale[soIndex] * val) + offset[soIndex];
469                    } else {
470                        new_values[k] = (scale[soIndex + k] * val) + offset[soIndex + k];
471                    }
472                } else {
473                    if (multiScaleDimLen == 1) {
474                        new_values[k] = scale[soIndex] * (val - offset[soIndex]);
475                    } else {
476                        new_values[k] = scale[soIndex + k] * (val - offset[soIndex + k]);
477                    }
478                }
479
480            } else {
481                new_values[k] = val;
482            }
483
484            // do valid range check AFTER scaling?
485            if (!rangeCheckBeforeScaling) {
486                if ((new_values[k] < valid_low) || (new_values[k] > valid_high)) {
487                    new_values[k] = Float.NaN;
488                }
489            }
490        }
491        return new_values;
492    }
493
494    /**
495     * Process a range of data from an array of {@code short} values.
496     * 
497     * @param values
498     *            Input {@code short} values. Cannot be {@code null}.
499     * @param subset
500     *            Optional subset.
501     *
502     * @return Processed range.
503     */
504    
505    public float[] processRange(short[] values, Map<String, double[]> subset) {
506
507        int multiScaleDimLen = 1;
508
509        if (subset != null) {
510            if (subset.get(multiScaleDimName) != null) {
511                double[] coords = subset.get(multiScaleDimName);
512                soIndex = (int) coords[0];
513                multiScaleDimLen = (int) (coords[1] - coords[0] + 1.0);
514            }
515        }
516
517        float[] new_values = new float[values.length];
518
519        float val = 0f;
520        int i = 0;
521        boolean isMissing = false;
522
523        for (int k = 0; k < values.length; k++) {
524
525            val = (float) values[k];
526            if (unsigned) {
527                i = Util.unsignedShortToInt(values[k]);
528                val = (float) i;
529            }
530
531            // first, check the (possibly multiple) missing values
532            isMissing = false;
533            if (missing != null) {
534                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
535                    if (val == missing[mvIdx]) {
536                        isMissing = true;
537                        break;
538                    }
539                }
540            }
541
542            if (isMissing) {
543                new_values[k] = Float.NaN;
544                continue;
545            }
546
547            if (rangeCheckBeforeScaling) {
548                if ((val < valid_low) || (val > valid_high)) {
549                    new_values[k] = Float.NaN;
550                    continue;
551                }
552            }
553
554            if (scale != null) {
555                if (unpack) {
556                    if (multiScaleDimLen == 1) {
557                        new_values[k] = (scale[soIndex] * val) + offset[soIndex];
558                    } else {
559                        new_values[k] = (scale[soIndex + k] * val) + offset[soIndex + k];
560                    }
561                } else {
562                    if (multiScaleDimLen == 1) {
563                        new_values[k] = scale[soIndex] * (val - offset[soIndex]);
564                    } else {
565
566                        new_values[k] = scale[soIndex + k] * (val - offset[soIndex + k]);
567                    }
568                }
569            } else {
570                new_values[k] = val;
571            }
572
573            // do valid range check AFTER scaling?
574            if (!rangeCheckBeforeScaling) {
575                if ((new_values[k] < valid_low) || (new_values[k] > valid_high)) {
576                    new_values[k] = Float.NaN;
577                }
578            }
579
580        }
581        return new_values;
582    }
583
584    /**
585     * Process a range of data from an array of {@code float} values.
586     * 
587     * @param values
588     *            Input {@code float} values. Cannot be {@code null}.
589     * @param subset
590     *            Optional subset.
591     *
592     * @return Processed array.
593     */
594    
595    public float[] processRange(float[] values, Map<String, double[]> subset) {
596
597        float[] new_values = null;
598
599        if ((missing != null) || (valid_range != null)) {
600            new_values = new float[values.length];
601        } else {
602            return values;
603        }
604
605        float val;
606
607        for (int k = 0; k < values.length; k++) {
608            val = values[k];
609            new_values[k] = val;
610
611            // first, check the (possibly multiple) missing values
612            if (missing != null) {
613                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
614                    if (val == missing[mvIdx]) {
615                        new_values[k] = Float.NaN;
616                        break;
617                    }
618                }
619            }
620
621            if ((valid_range != null) && ((val < valid_low) || (val > valid_high))) {
622                new_values[k] = Float.NaN;
623            }
624
625        }
626
627        return new_values;
628    }
629
630    /**
631     * Process a range of data from an array of {@code double} value.
632     * 
633     * @param values
634     *            Input {@code double} values. Cannot be {@code null}.
635     * @param subset
636     *            Optional subset.
637     *
638     * @return Processed array.
639     */
640    
641    public double[] processRange(double[] values, Map<String, double[]> subset) {
642
643        double[] new_values = null;
644
645        if ((missing != null) || (valid_range != null)) {
646            new_values = new double[values.length];
647        } else {
648            return values;
649        }
650
651        double val;
652
653        for (int k = 0; k < values.length; k++) {
654            val = values[k];
655            new_values[k] = val;
656
657            // first, check the (possibly multiple) missing values
658            if (missing != null) {
659                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
660                    if (val == missing[mvIdx]) {
661                        new_values[k] = Float.NaN;
662                        break;
663                    }
664                }
665            }
666
667            if ((valid_range != null) && ((val < valid_low) || (val > valid_high))) {
668                new_values[k] = Double.NaN;
669            }
670        }
671
672        return new_values;
673    }
674
675    public void setMultiScaleDimName(String multiScaleDimName) {
676        this.multiScaleDimName = multiScaleDimName;
677    }
678
679    public int getMultiScaleDimensionIndex() {
680        return multiScaleDimensionIndex;
681    }
682
683    public boolean hasMultiDimensionScale() {
684        return hasMultiDimensionScale;
685    }
686
687    public void setHasMultiDimensionScale(boolean yesno) {
688        hasMultiDimensionScale = yesno;
689    }
690
691    public void setMultiScaleIndex(int idx) {
692        this.soIndex = idx;
693    }
694
695    /**
696     * Should be generalized. For now works for short to float conversions
697     * 
698     * @param values
699     *            the set of input values to map
700     * @param lut
701     *            the lookup table for direct mapping input to output values
702     * @return output array as primitive floats
703     */
704
705    public Object processRangeApplyLUT(short[] values, float[] lut) {
706
707        float[] newValues = new float[values.length];
708
709        int lutLen = lut.length;
710
711        for (int i = 0; i < values.length; i++) {
712            short tmpVal = values[i];
713            if (tmpVal > 0) {
714                if (tmpVal < lutLen) {
715                    newValues[i] = lut[tmpVal];
716                } else {
717                    newValues[i] = Float.NaN;
718                }
719            } else {
720                newValues[i] = Float.NaN;
721            }
722        }
723
724        return newValues;
725    }
726
727}