001 /*
002 * $Id: GranuleAggregation.java,v 1.23 2012/04/10 15:32:42 rink Exp $
003 *
004 * This file is part of McIDAS-V
005 *
006 * Copyright 2007-2012
007 * Space Science and Engineering Center (SSEC)
008 * University of Wisconsin - Madison
009 * 1225 W. Dayton Street, Madison, WI 53706, USA
010 * https://www.ssec.wisc.edu/mcidas
011 *
012 * All Rights Reserved
013 *
014 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
015 * some McIDAS-V source code is based on IDV and VisAD source code.
016 *
017 * McIDAS-V is free software; you can redistribute it and/or modify
018 * it under the terms of the GNU Lesser Public License as published by
019 * the Free Software Foundation; either version 3 of the License, or
020 * (at your option) any later version.
021 *
022 * McIDAS-V is distributed in the hope that it will be useful,
023 * but WITHOUT ANY WARRANTY; without even the implied warranty of
024 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
025 * GNU Lesser Public License for more details.
026 *
027 * You should have received a copy of the GNU Lesser Public License
028 * along with this program. If not, see http://www.gnu.org/licenses.
029 */
030
031 package edu.wisc.ssec.mcidasv.data.hydra;
032
033 import java.util.ArrayList;
034 import java.util.HashMap;
035 import java.util.Iterator;
036 import java.util.List;
037
038 import org.slf4j.Logger;
039 import org.slf4j.LoggerFactory;
040
041 import ucar.ma2.Array;
042 import ucar.ma2.DataType;
043 import ucar.ma2.Range;
044 import ucar.ma2.StructureData;
045 import ucar.ma2.StructureMembers;
046
047 import ucar.nc2.Attribute;
048 import ucar.nc2.Dimension;
049 import ucar.nc2.NetcdfFile;
050 import ucar.nc2.Structure;
051 import ucar.nc2.Variable;
052
053 /**
054 * This file should be renamed - think about that Tommy.
055 * This file needs to implement the same signatures NetCDFFile does,
056 * but for aggregations of consecutive granules.
057 *
058 * @author tommyj
059 *
060 */
061
062 public class GranuleAggregation implements MultiDimensionReader {
063
064 private static final Logger logger = LoggerFactory.getLogger(GranuleAggregation.class);
065
066 // this structure holds the NcML readers that get passed in
067 ArrayList<NetcdfFile> nclist = new ArrayList<NetcdfFile>();
068
069 // this holds the MultiDimensionReaders, here NetCDFFile
070 ArrayList<NetCDFFile> ncdfal = null;
071
072 // need an ArrayList for each variable hashmap structure
073 ArrayList<HashMap<String, Variable>> varMapList = new ArrayList<HashMap<String, Variable>>();
074 ArrayList<HashMap<String, String[]>> varDimNamesList = new ArrayList<HashMap<String, String[]>>();
075 ArrayList<HashMap<String, int[]>> varDimLengthsList = new ArrayList<HashMap<String, int[]>>();
076 ArrayList<HashMap<String, Class>> varDataTypeList = new ArrayList<HashMap<String, Class>>();
077 ArrayList<HashMap<String, Integer>> varGranuleRatiosList = new ArrayList<HashMap<String, Integer>>();
078
079 // variable can have bulk array processor set by the application
080 HashMap<String, RangeProcessor> varToRangeProcessor = new HashMap<String, RangeProcessor>();
081
082 private int granuleCount = -1;
083 private int granuleLength = -1;
084 private String inTrackDimensionName = null;
085 private String crossTrackDimensionName = null;
086
087 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, int granuleLength, String inTrackDimensionName, String crossTrackDimensionName) throws Exception {
088 if (ncdfal == null) throw new Exception("No data: empty Suomi NPP aggregation object");
089 logger.trace("granule length: " + granuleLength + " inTrack: " + inTrackDimensionName);
090 this.granuleLength = granuleLength;
091 this.inTrackDimensionName = inTrackDimensionName;
092 this.crossTrackDimensionName = crossTrackDimensionName;
093 this.ncdfal = ncdfal;
094 init(ncdfal);
095 }
096
097 public Class getArrayType(String array_name) {
098 return varDataTypeList.get(0).get(array_name);
099 }
100
101 public String[] getDimensionNames(String array_name) {
102 logger.trace("GranuleAggregation.getDimensionNames, requested: " + array_name);
103 return varDimNamesList.get(0).get(array_name);
104 }
105
106 public int[] getDimensionLengths(String array_name) {
107 logger.trace("GranuleAggregation.getDimensionLengths, requested: " + array_name);
108 int[] lengths = varDimLengthsList.get(0).get(array_name);
109 for (int i = 0; i < lengths.length; i++) {
110 logger.trace("Length: " + lengths[i]);
111 }
112 return varDimLengthsList.get(0).get(array_name);
113 }
114
115 public float[] getFloatArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
116 return (float[]) readArray(array_name, start, count, stride);
117 }
118
119 public int[] getIntArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
120 return (int[]) readArray(array_name, start, count, stride);
121 }
122
123 public double[] getDoubleArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
124 return (double[]) readArray(array_name, start, count, stride);
125 }
126
127 public short[] getShortArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
128 return (short[]) readArray(array_name, start, count, stride);
129 }
130
131 public byte[] getByteArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
132 return (byte[]) readArray(array_name, start, count, stride);
133 }
134
135 public Object getArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
136 return readArray(array_name, start, count, stride);
137 }
138
139 public HDFArray getGlobalAttribute(String attr_name) throws Exception {
140 throw new Exception("GranuleAggregation.getGlobalAttributes: Unimplemented");
141 }
142
143 public HDFArray getArrayAttribute(String array_name, String attr_name) throws Exception {
144 Variable var = varMapList.get(0).get(array_name);
145 if (var == null) return null;
146
147 Attribute attr = var.findAttribute(attr_name);
148 if (attr == null) return null;
149
150 Array attrVals = attr.getValues();
151 DataType dataType = attr.getDataType();
152 Object array = attrVals.copyTo1DJavaArray();
153
154 HDFArray harray = null;
155
156 if (dataType.getPrimitiveClassType() == Float.TYPE) {
157 harray = HDFArray.make((float[])array);
158 }
159 else if (dataType.getPrimitiveClassType() == Double.TYPE) {
160 harray = HDFArray.make((double[])array);
161 }
162 else if (dataType == DataType.STRING) {
163 harray = HDFArray.make((String[])array);
164 }
165 else if (dataType.getPrimitiveClassType() == Short.TYPE) {
166 harray = HDFArray.make((short[])array);
167 }
168 else if (dataType.getPrimitiveClassType() == Integer.TYPE) {
169 harray = HDFArray.make((int[])array);
170 }
171 return harray;
172 }
173
174 public void close() throws Exception {
175 // close each NetCDF file
176 for (NetcdfFile n : nclist) {
177 n.close();
178 }
179 }
180
181 private void init(ArrayList<NetCDFFile> ncdfal) throws Exception {
182
183 logger.debug("init in...");
184 // make a NetCDFFile object from the NcML for each granule
185 for (NetCDFFile n : ncdfal) {
186 logger.debug("loading another NetCDF file from NcML...");
187 NetcdfFile ncfile = n.getNetCDFFile();
188 nclist.add(ncfile);
189 }
190
191 granuleCount = nclist.size();
192 logger.debug("Granule count: " + granuleCount);
193
194 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) {
195
196 NetcdfFile ncfile = nclist.get(ncIdx);
197
198 HashMap<String, Variable> varMap = new HashMap<String, Variable>();
199 HashMap<String, String[]> varDimNames = new HashMap<String, String[]>();
200 HashMap<String, int[]> varDimLengths = new HashMap<String, int[]>();
201 HashMap<String, Class> varDataType = new HashMap<String, Class>();
202 HashMap<String, Integer> varGranuleRatios = new HashMap<String, Integer>();
203
204 Iterator varIter = ncfile.getVariables().iterator();
205 int varInTrackIndex = -1;
206 while (varIter.hasNext()) {
207 Variable var = (Variable) varIter.next();
208
209 logger.debug("Working on variable: " + var.getShortName());
210
211 if (var instanceof Structure) {
212 analyzeStructure((Structure) var, varMap, varDimNames, varDimLengths, varDataType);
213 continue;
214 }
215
216 int rank = var.getRank();
217
218 // bypass any less-than-2D variables for now...
219 if (rank < 2) {
220 logger.debug("Skipping 1D variable: " + var.getShortName());
221 continue;
222 }
223
224 String varName = var.getFullName();
225 varMap.put(varName, var);
226 Iterator dimIter = var.getDimensions().iterator();
227 String[] dimNames = new String[rank];
228 int[] dimLengths = new int[rank];
229 int cnt = 0;
230 boolean notDisplayable = false;
231 while (dimIter.hasNext()) {
232 Dimension dim = (Dimension) dimIter.next();
233 String s = dim.getName();
234 logger.trace("DIMENSION name: " + s);
235 if ((s != null) && (!s.isEmpty())) {
236 if ((! s.equals(inTrackDimensionName)) &&
237 ((! s.startsWith("Band")) && (cnt == 0)) &&
238 (! varName.endsWith("Latitude")) &&
239 (! varName.endsWith("Longitude")) &&
240 (! s.equals(crossTrackDimensionName))) {
241 notDisplayable = true;
242 break;
243 }
244 }
245 String dimName = dim.getName();
246 logger.trace("GranuleAggregation init, variable: " + varName + ", dimension name: " + dimName + ", length: " + dim.getLength());
247 if (dimName == null) dimName = "dim" + cnt;
248 dimNames[cnt] = dimName;
249 dimLengths[cnt] = dim.getLength();
250 cnt++;
251 }
252
253 // skip to next variable if it's not displayable data
254 if (notDisplayable) continue;
255
256 varInTrackIndex = getInTrackIndex(var);
257 logger.debug("Found in-track index of: " + varInTrackIndex);
258
259 if (varInTrackIndex < 0) {
260 logger.debug("Skipping variable with unknown dimension: " + var.getShortName());
261 continue;
262 }
263
264 // store the ratio of this variable's in-track length to the granule length
265 varGranuleRatios.put(varName, new Integer(granuleLength / dimLengths[varInTrackIndex]));
266
267 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] * granuleCount;
268
269 varDimNames.put(varName, dimNames);
270 varDimLengths.put(varName, dimLengths);
271 varDataType.put(varName, var.getDataType().getPrimitiveClassType());
272 }
273
274 // add the new hashmaps to our enclosing lists
275 varMapList.add(varMap);
276 varDimNamesList.add(varDimNames);
277 varDimLengthsList.add(varDimLengths);
278 varDataTypeList.add(varDataType);
279 varGranuleRatiosList.add(varGranuleRatios);
280 }
281 }
282
283 /**
284 * Based on the names of the variable dimensions, determine the in-track index
285 * @param dimNames names of dimensions - should match static strings in relevant classes
286 * @return correct index (0 or greater), or -1 if error
287 */
288
289 private int getInTrackIndex(Variable v) {
290
291 int index = -1;
292 boolean is2D = false;
293 boolean is3D = false;
294
295 String inTrackName = null;
296
297 // typical sanity check
298 if (v == null) return index;
299 logger.debug("getInTrackIndex called for variable: " + v.getShortName());
300
301 // lat/lon vars have different dimension names
302 if ((v.getFullName().endsWith("Latitude")) || (v.getFullName().endsWith("Longitude"))) {
303 if (v.getFullName().startsWith("All_Data")) {
304 inTrackName = inTrackDimensionName;
305 } else {
306 inTrackName = "2*nscans";
307 }
308 } else {
309 inTrackName = inTrackDimensionName;
310 }
311 // pull out the dimensions
312 List<Dimension> dList = v.getDimensions();
313
314 // right now, we only handle 2D and 3D variables.
315 // TJJ XXX it does get trickier, and we will have to expand this
316 // to deal with for example CrIS data...
317 int numDimensions = dList.size();
318 logger.debug("Number of dimensions: " + numDimensions);
319
320 // the only 4D data right now is CrIS, return 0
321 if (numDimensions == 4) return 0;
322
323 if ((numDimensions == 2) || (numDimensions == 3)) {
324 if (numDimensions == 2) is2D = true;
325 if (numDimensions == 3) is3D = true;
326 } else {
327 return index;
328 }
329
330 // if the data is 2D, we use the SwathAdapter class,
331 // if 3D, we use the SpectrumAdapter class
332 for (int i = 0; i < numDimensions; i++) {
333 if (is2D) {
334 // XXX TJJ - if empty name, in-track index is 0
335 if ((dList.get(i).getName() == null) || (dList.get(i).getName().isEmpty())) {
336 logger.debug("WARNING: Empty dimension name!, assuming in-track dim is 0");
337 return 0;
338 }
339 logger.debug("Comparing: " + dList.get(i).getName() + " with: " + inTrackName);
340 if (dList.get(i).getName().equals(inTrackName)) {
341 index = i;
342 break;
343 }
344 }
345 if (is3D) {
346 // XXX TJJ - if empty name, in-track index is 0
347 if ((dList.get(i).getName() == null) || (dList.get(i).getName().isEmpty())) {
348 logger.debug("WARNING: Empty dimension name!, assuming in-track dim is 0");
349 return 0;
350 }
351 logger.debug("Comparing: " + dList.get(i).getName() + " with: " + inTrackName);
352 if (dList.get(i).getName().equals(inTrackName)) {
353 index = i;
354 break;
355 }
356 }
357 }
358
359 // hopefully we found the right one
360 return index;
361 }
362
363 void analyzeStructure(Structure var, HashMap<String, Variable> varMap,
364 HashMap<String, String[]> varDimNames,
365 HashMap<String, int[]> varDimLengths,
366 HashMap<String, Class> varDataType) throws Exception {
367 if ((var.getShape()).length == 0) {
368 return;
369 }
370 String varName = var.getFullName();
371 String[] dimNames = new String[2];
372 int[] dimLengths = new int[2];
373 int cnt = 0;
374 dimLengths[0] = (var.getShape())[0];
375 dimNames[0] = "dim"+cnt;
376
377 cnt++;
378 StructureData sData = var.readStructure(0);
379 List memList = sData.getMembers();
380 dimLengths[1] = memList.size();
381 dimNames[1] = "dim"+cnt;
382
383 varDimNames.put(varName, dimNames);
384 varDimLengths.put(varName, dimLengths);
385 varMap.put(varName, var);
386
387 StructureMembers sMembers = sData.getStructureMembers();
388 Object obj = sData.getScalarObject(sMembers.getMember(0));
389 varDataType.put(varName, obj.getClass());
390 }
391
392 private synchronized Object readArray(String array_name, int[] start, int[] count, int[] stride) throws Exception {
393
394 // how many dimensions are we dealing with
395 int dimensionCount = start.length;
396
397 // pull out a representative variable so we can determine which index is in-track
398 Variable vTmp = varMapList.get(0).get(array_name);
399 int vInTrackIndex = getInTrackIndex(vTmp);
400
401 int vGranuleRatio = varGranuleRatiosList.get(0).get(array_name);
402 int vGranuleLength = granuleLength / vGranuleRatio;
403
404 logger.trace("READING: " + array_name + ", INTRACKINDEX: " + vInTrackIndex +
405 ", RATIO: " + vGranuleRatio);
406
407 // which granules will we be dealing with?
408 int loGranuleId = start[vInTrackIndex] / vGranuleLength;
409 int hiGranuleId = (start[vInTrackIndex] + ((count[vInTrackIndex] - 1) * stride[vInTrackIndex])) / vGranuleLength;
410
411 // next, we break out the offsets, counts, and strides for each granule
412 int granuleSpan = hiGranuleId - loGranuleId + 1;
413 logger.trace("readArray req, loGran: " + loGranuleId + ", hiGran: " +
414 hiGranuleId + ", granule span: " + granuleSpan + ", dimCount: " + dimensionCount);
415 for (int i = 0; i < dimensionCount; i++) {
416 logger.trace("start[" + i + "]: " + start[i]);
417 logger.trace("count[" + i + "]: " + count[i]);
418 logger.trace("stride[" + i + "]: " + stride[i]);
419 }
420 int [][] startSet = new int [granuleSpan][dimensionCount];
421 int [][] countSet = new int [granuleSpan][dimensionCount];
422 int [][] strideSet = new int [granuleSpan][dimensionCount];
423 int countSubtotal = 0;
424
425 // this part is a little tricky - set the values for each granule we need to access for this read
426 int granuleNumber = loGranuleId;
427 for (int i = 0; i < granuleSpan; i++) {
428 granuleNumber++;
429 for (int j = 0; j < dimensionCount; j++) {
430 // for all indeces other than the in-track index, the numbers match what was passed in
431 if (j != vInTrackIndex) {
432 startSet[i][j] = start[j];
433 countSet[i][j] = count[j] * stride[j];
434 strideSet[i][j] = stride[j];
435 } else {
436 // for the in-track index, it's not so easy...
437 // for first granule, start is what's passed in
438 if (i == 0) {
439 startSet[i][j] = start[j] % vGranuleLength;
440 } else {
441 startSet[i][j] = ((vGranuleLength * granuleNumber) - start[j]) % stride[j];
442 }
443 // counts may be different for start, end, and middle granules
444 if (i == 0) {
445 // is this the first and only granule?
446 if (granuleSpan == 1) {
447 countSet[i][j] = count[j] * stride[j];
448 // or is this the first of multiple granules...
449 } else {
450 if (((vGranuleLength * granuleNumber) - start[j]) < (count[j] * stride[j])) {
451 countSet[i][j] = ((vGranuleLength * granuleNumber) - start[j]);
452 } else {
453 countSet[i][j] = count[j] * stride[j];
454 }
455 countSubtotal += countSet[i][j];
456 }
457 } else {
458 // middle granules
459 if (i < (granuleSpan - 1)) {
460 countSet[i][j] = vGranuleLength;
461 countSubtotal += countSet[i][j];
462 } else {
463 // the end granule
464 countSet[i][j] = (count[j] * stride[j]) - countSubtotal;
465 // XXX TJJ - limiting count to valid numbers here, why??
466 // need to revisit, see why this condition manifests
467 if (countSet[i][j] > (vGranuleLength - startSet[i][j]))
468 countSet[i][j] = vGranuleLength - startSet[i][j];
469 }
470 }
471 // luckily, stride never changes
472 strideSet[i][j] = stride[j];
473 }
474 }
475 }
476
477 int totalLength = 0;
478 int rangeListCount = 0;
479 ArrayList<Array> arrayList = new ArrayList<Array>();
480 for (int granuleIdx = 0; granuleIdx < granuleCount; granuleIdx++) {
481 if ((granuleIdx >= loGranuleId) && (granuleIdx <= hiGranuleId)) {
482 Variable var = varMapList.get(loGranuleId + (granuleIdx-loGranuleId)).get(array_name);
483
484 if (var instanceof Structure) {
485 // what to do here?
486 } else {
487 ArrayList<Range> rangeList = new ArrayList<Range>();
488 for (int dimensionIdx = 0; dimensionIdx < dimensionCount; dimensionIdx++) {
489 logger.trace("Creating new Range: " + startSet[rangeListCount][dimensionIdx] +
490 ", " + (startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1) + ", " + strideSet[rangeListCount][dimensionIdx]);
491 Range range = new Range(
492 startSet[rangeListCount][dimensionIdx],
493 startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1,
494 strideSet[rangeListCount][dimensionIdx]
495 );
496 rangeList.add(dimensionIdx, range);
497 }
498 rangeListCount++;
499 Array subarray = var.read(rangeList);
500 //dataType = subarray.getElementType();
501 totalLength += subarray.getSize();
502 arrayList.add(subarray);
503 }
504 // put in an empty ArrayList placeholder to retain a slot for each granule
505 } else {
506 Array emptyArray = null;
507 arrayList.add(emptyArray);
508 }
509 }
510
511 // last, concatenate the individual NetCDF arrays pulled out
512
513 Class outType;
514 Class arrayType = getArrayType(array_name);
515 RangeProcessor rngProcessor = varToRangeProcessor.get(array_name);
516 if (rngProcessor == null) {
517 outType = getArrayType(array_name);
518 }
519 else {
520 outType = java.lang.Float.TYPE;
521 }
522 Object o = java.lang.reflect.Array.newInstance(outType, totalLength);
523
524 int destPos = 0;
525 int granIdx = 0;
526
527 for (Array a : arrayList) {
528 if (a != null) {
529 Object primArray = a.copyTo1DJavaArray();
530 primArray = processArray(array_name, arrayType, granIdx, primArray, rngProcessor, start, count);
531 System.arraycopy(primArray, 0, o, destPos, (int) a.getSize());
532 destPos += a.getSize();
533 }
534 granIdx++;
535 }
536
537
538 return o;
539 }
540
541 public HashMap getVarMap() {
542 return varMapList.get(0);
543 }
544
545 public ArrayList<NetCDFFile> getReaders() {
546 return this.ncdfal;
547 }
548
549 /* pass individual granule pieces just read from dataset through the RangeProcessor */
550 private Object processArray(String array_name, Class arrayType, int granIdx, Object values, RangeProcessor rngProcessor, int[] start, int[] count) {
551 if (rngProcessor == null) {
552 return values;
553 }
554 else {
555 ((AggregationRangeProcessor)rngProcessor).setWhichRangeProcessor(granIdx);
556
557 boolean processAlongMultiScaleDim = false;
558
559 if (rngProcessor.hasMultiDimensionScale()) { // this data variable has an array > 1 of scale/offsets. For example, one for each band.
560 rngProcessor.setMultiScaleIndex(start[rngProcessor.getMultiScaleDimensionIndex()]);
561 if (count[rngProcessor.getMultiScaleDimensionIndex()] > 1) { // if the multiScaleDim is > 1, use processAlongMultiScaleDim below
562 processAlongMultiScaleDim = true;
563 }
564 }
565
566 Object outArray = null;
567
568 if (processAlongMultiScaleDim) {
569
570 if (arrayType == Short.TYPE) {
571 outArray = rngProcessor.processAlongMultiScaleDim((short[])values);
572 } else if (arrayType == Byte.TYPE) {
573 outArray = rngProcessor.processAlongMultiScaleDim((byte[])values);
574 } else if (arrayType == Float.TYPE) {
575 outArray = values;
576 } else if (arrayType == Double.TYPE) {
577 outArray = values;
578 }
579
580 }
581 else {
582
583 if (arrayType == Short.TYPE) {
584 outArray = rngProcessor.processRange((short[])values, null);
585 } else if (arrayType == Byte.TYPE) {
586 outArray = rngProcessor.processRange((byte[])values, null);
587 } else if (arrayType == Float.TYPE) {
588 outArray = rngProcessor.processRange((float[])values, null);
589 } else if (arrayType == Double.TYPE) {
590 outArray = rngProcessor.processRange((double[])values, null);
591 }
592
593 }
594
595 return outArray;
596 }
597 }
598
599 /* Application can supply a RangeProcessor for an variable 'arrayName' */
600 public void addRangeProcessor(String arrayName, RangeProcessor rangeProcessor) {
601 varToRangeProcessor.put(arrayName, rangeProcessor);
602 }
603 }