001/* 002 * $Id: GranuleAggregation.java,v 1.17 2011/03/24 16:06:33 davep Exp $ 003 * 004 * This file is part of McIDAS-V 005 * 006 * Copyright 2007-2011 007 * Space Science and Engineering Center (SSEC) 008 * University of Wisconsin - Madison 009 * 1225 W. Dayton Street, Madison, WI 53706, USA 010 * https://www.ssec.wisc.edu/mcidas 011 * 012 * All Rights Reserved 013 * 014 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 015 * some McIDAS-V source code is based on IDV and VisAD source code. 016 * 017 * McIDAS-V is free software; you can redistribute it and/or modify 018 * it under the terms of the GNU Lesser Public License as published by 019 * the Free Software Foundation; either version 3 of the License, or 020 * (at your option) any later version. 021 * 022 * McIDAS-V is distributed in the hope that it will be useful, 023 * but WITHOUT ANY WARRANTY; without even the implied warranty of 024 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 025 * GNU Lesser Public License for more details. 026 * 027 * You should have received a copy of the GNU Lesser Public License 028 * along with this program. If not, see http://www.gnu.org/licenses. 029 */ 030 031package edu.wisc.ssec.mcidasv.data.hydra; 032 033import java.util.ArrayList; 034import java.util.HashMap; 035import java.util.Iterator; 036import java.util.List; 037 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041import ucar.ma2.Array; 042import ucar.ma2.DataType; 043import ucar.ma2.Range; 044import ucar.ma2.StructureData; 045import ucar.ma2.StructureMembers; 046 047import ucar.nc2.Attribute; 048import ucar.nc2.Dimension; 049import ucar.nc2.NetcdfFile; 050import ucar.nc2.Structure; 051import ucar.nc2.Variable; 052 053/** 054 * This file should be renamed - think about that Tommy. 055 * This file needs to implement the same signatures NetCDFFile does, 056 * but for aggregations of consecutive granules. 057 * 058 * @author tommyj 059 * 060 */ 061 062public class GranuleAggregation implements MultiDimensionReader { 063 064 private static final Logger logger = LoggerFactory.getLogger(GranuleAggregation.class); 065 066 // this structure holds the NcML readers that get passed in 067 ArrayList<NetcdfFile> nclist = new ArrayList<NetcdfFile>(); 068 069 // this holds the MultiDimensionReaders, here NetCDFFile 070 ArrayList<NetCDFFile> ncdfal = null; 071 072 // need an ArrayList for each variable hashmap structure 073 ArrayList<HashMap<String, Variable>> varMapList = new ArrayList<HashMap<String, Variable>>(); 074 ArrayList<HashMap<String, String[]>> varDimNamesList = new ArrayList<HashMap<String, String[]>>(); 075 ArrayList<HashMap<String, int[]>> varDimLengthsList = new ArrayList<HashMap<String, int[]>>(); 076 ArrayList<HashMap<String, Class>> varDataTypeList = new ArrayList<HashMap<String, Class>>(); 077 ArrayList<HashMap<String, Integer>> varGranuleRatiosList = new ArrayList<HashMap<String, Integer>>(); 078 079 // variable can have bulk array processor set by the application 080 HashMap<String, RangeProcessor> varToRangeProcessor = new HashMap<String, RangeProcessor>(); 081 082 private int granuleCount = -1; 083 private int granuleLength = -1; 084 private String inTrackDimensionName = null; 085 private String crossTrackDimensionName = null; 086 087 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, int granuleLength, String inTrackDimensionName, String crossTrackDimensionName) throws Exception { 088 if (ncdfal == null) throw new Exception("No data: empty NPP aggregation object"); 089 logger.trace("granule length: " + granuleLength + " inTrack: " + inTrackDimensionName); 090 this.granuleLength = granuleLength; 091 this.inTrackDimensionName = inTrackDimensionName; 092 this.crossTrackDimensionName = crossTrackDimensionName; 093 this.ncdfal = ncdfal; 094 init(ncdfal); 095 } 096 097 public Class getArrayType(String array_name) { 098 return varDataTypeList.get(0).get(array_name); 099 } 100 101 public String[] getDimensionNames(String array_name) { 102 logger.trace("GranuleAggregation.getDimensionNames, requested: " + array_name); 103 return varDimNamesList.get(0).get(array_name); 104 } 105 106 public int[] getDimensionLengths(String array_name) { 107 logger.trace("GranuleAggregation.getDimensionLengths, requested: " + array_name); 108 int[] lengths = varDimLengthsList.get(0).get(array_name); 109 for (int i = 0; i < lengths.length; i++) { 110 logger.trace("Length: " + lengths[i]); 111 } 112 return varDimLengthsList.get(0).get(array_name); 113 } 114 115 public float[] getFloatArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 116 return (float[]) readArray(array_name, start, count, stride); 117 } 118 119 public int[] getIntArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 120 return (int[]) readArray(array_name, start, count, stride); 121 } 122 123 public double[] getDoubleArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 124 return (double[]) readArray(array_name, start, count, stride); 125 } 126 127 public short[] getShortArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 128 return (short[]) readArray(array_name, start, count, stride); 129 } 130 131 public byte[] getByteArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 132 return (byte[]) readArray(array_name, start, count, stride); 133 } 134 135 public Object getArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 136 return readArray(array_name, start, count, stride); 137 } 138 139 public HDFArray getGlobalAttribute(String attr_name) throws Exception { 140 throw new Exception("GranuleAggregation.getGlobalAttributes: Unimplemented"); 141 } 142 143 public HDFArray getArrayAttribute(String array_name, String attr_name) throws Exception { 144 Variable var = varMapList.get(0).get(array_name); 145 if (var == null) return null; 146 147 Attribute attr = var.findAttribute(attr_name); 148 if (attr == null) return null; 149 150 Array attrVals = attr.getValues(); 151 DataType dataType = attr.getDataType(); 152 Object array = attrVals.copyTo1DJavaArray(); 153 154 HDFArray harray = null; 155 156 if (dataType.getPrimitiveClassType() == Float.TYPE) { 157 harray = HDFArray.make((float[])array); 158 } 159 else if (dataType.getPrimitiveClassType() == Double.TYPE) { 160 harray = HDFArray.make((double[])array); 161 } 162 else if (dataType == DataType.STRING) { 163 harray = HDFArray.make((String[])array); 164 } 165 else if (dataType.getPrimitiveClassType() == Short.TYPE) { 166 harray = HDFArray.make((short[])array); 167 } 168 else if (dataType.getPrimitiveClassType() == Integer.TYPE) { 169 harray = HDFArray.make((int[])array); 170 } 171 return harray; 172 } 173 174 public void close() throws Exception { 175 // close each NetCDF file 176 for (NetcdfFile n : nclist) { 177 n.close(); 178 } 179 } 180 181 private void init(ArrayList<NetCDFFile> ncdfal) throws Exception { 182 183 logger.debug("init in..."); 184 // make a NetCDFFile object from the NcML for each granule 185 for (NetCDFFile n : ncdfal) { 186 logger.debug("loading another NetCDF file from NcML..."); 187 NetcdfFile ncfile = n.getNetCDFFile(); 188 nclist.add(ncfile); 189 } 190 191 granuleCount = nclist.size(); 192 logger.debug("Granule count: " + granuleCount); 193 194 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 195 196 NetcdfFile ncfile = nclist.get(ncIdx); 197 198 HashMap<String, Variable> varMap = new HashMap<String, Variable>(); 199 HashMap<String, String[]> varDimNames = new HashMap<String, String[]>(); 200 HashMap<String, int[]> varDimLengths = new HashMap<String, int[]>(); 201 HashMap<String, Class> varDataType = new HashMap<String, Class>(); 202 HashMap<String, Integer> varGranuleRatios = new HashMap<String, Integer>(); 203 204 Iterator varIter = ncfile.getVariables().iterator(); 205 int varInTrackIndex = -1; 206 while (varIter.hasNext()) { 207 Variable var = (Variable) varIter.next(); 208 209 logger.debug("Working on variable: " + var.getName()); 210 211 if (var instanceof Structure) { 212 analyzeStructure((Structure) var, varMap, varDimNames, varDimLengths, varDataType); 213 continue; 214 } 215 216 int rank = var.getRank(); 217 218 // bypass any less-than-2D variables for now... 219 if (rank < 2) { 220 logger.debug("Skipping 1D variable: " + var.getName()); 221 continue; 222 } 223 224 String varName = var.getName(); 225 varMap.put(var.getName(), var); 226 Iterator dimIter = var.getDimensions().iterator(); 227 String[] dimNames = new String[rank]; 228 int[] dimLengths = new int[rank]; 229 int cnt = 0; 230 boolean notDisplayable = false; 231 while (dimIter.hasNext()) { 232 Dimension dim = (Dimension) dimIter.next(); 233 String s = dim.getName(); 234 logger.trace("DIMENSION name: " + s); 235 if ((s != null) && (!s.isEmpty())) { 236 if ((! s.equals(inTrackDimensionName)) && 237 ((! s.startsWith("Band")) && (cnt == 0)) && 238 (! var.getName().endsWith("Latitude")) && 239 (! var.getName().endsWith("Longitude")) && 240 (! s.equals(crossTrackDimensionName))) { 241 notDisplayable = true; 242 break; 243 } 244 } 245 String dimName = dim.getName(); 246 logger.trace("GranuleAggregation init, variable: " + varName + ", dimension name: " + dimName + ", length: " + dim.getLength()); 247 if (dimName == null) dimName = "dim" + cnt; 248 dimNames[cnt] = dimName; 249 dimLengths[cnt] = dim.getLength(); 250 cnt++; 251 } 252 253 // skip to next variable if it's not displayable data 254 if (notDisplayable) continue; 255 256 varInTrackIndex = getInTrackIndex(var); 257 logger.debug("Found in-track index of: " + varInTrackIndex); 258 259 if (varInTrackIndex < 0) { 260 logger.debug("Skipping variable with unknown dimension: " + var.getName()); 261 continue; 262 } 263 264 // store the ratio of this variable's in-track length to the granule length 265 varGranuleRatios.put(varName, new Integer(granuleLength / dimLengths[varInTrackIndex])); 266 267 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] * granuleCount; 268 269 varDimNames.put(varName, dimNames); 270 varDimLengths.put(varName, dimLengths); 271 varDataType.put(varName, var.getDataType().getPrimitiveClassType()); 272 } 273 274 // add the new hashmaps to our enclosing lists 275 varMapList.add(varMap); 276 varDimNamesList.add(varDimNames); 277 varDimLengthsList.add(varDimLengths); 278 varDataTypeList.add(varDataType); 279 varGranuleRatiosList.add(varGranuleRatios); 280 } 281 } 282 283 /** 284 * Based on the names of the variable dimensions, determine the in-track index 285 * @param dimNames names of dimensions - should match static strings in relevant classes 286 * @return correct index (0 or greater), or -1 if error 287 */ 288 289 private int getInTrackIndex(Variable v) { 290 291 int index = -1; 292 boolean is2D = false; 293 boolean is3D = false; 294 295 String inTrackName = null; 296 297 // typical sanity check 298 if (v == null) return index; 299 logger.debug("getInTrackIndex called for variable: " + v.getName()); 300 301 // lat/lon vars have different dimension names 302 if ((v.getName().endsWith("Latitude")) || (v.getName().endsWith("Longitude"))) { 303 if (v.getName().startsWith("All_Data")) { 304 inTrackName = inTrackDimensionName; 305 } else { 306 inTrackName = "2*nscans"; 307 } 308 } else { 309 inTrackName = inTrackDimensionName; 310 } 311 // pull out the dimensions 312 List<Dimension> dList = v.getDimensions(); 313 314 // right now, we only handle 2D and 3D variables. 315 // TJJ XXX it does get trickier, and we will have to expand this 316 // to deal with for example CrIS data... 317 int numDimensions = dList.size(); 318 logger.debug("Number of dimensions: " + numDimensions); 319 320 // the only 4D data right now is CrIS, return 0 321 if (numDimensions == 4) return 0; 322 323 if ((numDimensions == 2) || (numDimensions == 3)) { 324 if (numDimensions == 2) is2D = true; 325 if (numDimensions == 3) is3D = true; 326 } else { 327 return index; 328 } 329 330 // if the data is 2D, we use the SwathAdapter class, 331 // if 3D, we use the SpectrumAdapter class 332 for (int i = 0; i < numDimensions; i++) { 333 if (is2D) { 334 // XXX TJJ - if empty name, in-track index is 0 335 if ((dList.get(i).getName() == null) || (dList.get(i).getName().isEmpty())) { 336 logger.debug("WARNING: Empty dimension name!, assuming in-track dim is 0"); 337 return 0; 338 } 339 logger.debug("Comparing: " + dList.get(i).getName() + " with: " + inTrackName); 340 if (dList.get(i).getName().equals(inTrackName)) { 341 index = i; 342 break; 343 } 344 } 345 if (is3D) { 346 // XXX TJJ - if empty name, in-track index is 0 347 if ((dList.get(i).getName() == null) || (dList.get(i).getName().isEmpty())) { 348 logger.debug("WARNING: Empty dimension name!, assuming in-track dim is 0"); 349 return 0; 350 } 351 logger.debug("Comparing: " + dList.get(i).getName() + " with: " + inTrackName); 352 if (dList.get(i).getName().equals(inTrackName)) { 353 index = i; 354 break; 355 } 356 } 357 } 358 359 // hopefully we found the right one 360 return index; 361 } 362 363 void analyzeStructure(Structure var, HashMap<String, Variable> varMap, 364 HashMap<String, String[]> varDimNames, 365 HashMap<String, int[]> varDimLengths, 366 HashMap<String, Class> varDataType) throws Exception { 367 if ((var.getShape()).length == 0) { 368 return; 369 } 370 String varName = var.getName(); 371 String[] dimNames = new String[2]; 372 int[] dimLengths = new int[2]; 373 int cnt = 0; 374 dimLengths[0] = (var.getShape())[0]; 375 dimNames[0] = "dim"+cnt; 376 377 cnt++; 378 StructureData sData = var.readStructure(0); 379 List memList = sData.getMembers(); 380 dimLengths[1] = memList.size(); 381 dimNames[1] = "dim"+cnt; 382 383 varDimNames.put(varName, dimNames); 384 varDimLengths.put(varName, dimLengths); 385 varMap.put(var.getName(), var); 386 387 StructureMembers sMembers = sData.getStructureMembers(); 388 Object obj = sData.getScalarObject(sMembers.getMember(0)); 389 varDataType.put(varName, obj.getClass()); 390 } 391 392 private synchronized Object readArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 393 394 // how many dimensions are we dealing with 395 int dimensionCount = start.length; 396 397 // pull out a representative variable so we can determine which index is in-track 398 Variable vTmp = varMapList.get(0).get(array_name); 399 int vInTrackIndex = getInTrackIndex(vTmp); 400 401 int vGranuleRatio = varGranuleRatiosList.get(0).get(array_name); 402 int vGranuleLength = granuleLength / vGranuleRatio; 403 404 logger.trace("READING: " + array_name + ", INTRACKINDEX: " + vInTrackIndex + 405 ", RATIO: " + vGranuleRatio); 406 407 // which granules will we be dealing with? 408 int loGranuleId = start[vInTrackIndex] / vGranuleLength; 409 int hiGranuleId = (start[vInTrackIndex] + ((count[vInTrackIndex] - 1) * stride[vInTrackIndex])) / vGranuleLength; 410 411 // next, we break out the offsets, counts, and strides for each granule 412 int granuleSpan = hiGranuleId - loGranuleId + 1; 413 logger.trace("readArray req, loGran: " + loGranuleId + ", hiGran: " + 414 hiGranuleId + ", granule span: " + granuleSpan + ", dimCount: " + dimensionCount); 415 for (int i = 0; i < dimensionCount; i++) { 416 logger.trace("start[" + i + "]: " + start[i]); 417 logger.trace("count[" + i + "]: " + count[i]); 418 logger.trace("stride[" + i + "]: " + stride[i]); 419 } 420 int [][] startSet = new int [granuleSpan][dimensionCount]; 421 int [][] countSet = new int [granuleSpan][dimensionCount]; 422 int [][] strideSet = new int [granuleSpan][dimensionCount]; 423 int countSubtotal = 0; 424 425 // this part is a little tricky - set the values for each granule we need to access for this read 426 int granuleNumber = loGranuleId; 427 for (int i = 0; i < granuleSpan; i++) { 428 granuleNumber++; 429 for (int j = 0; j < dimensionCount; j++) { 430 // for all indeces other than the in-track index, the numbers match what was passed in 431 if (j != vInTrackIndex) { 432 startSet[i][j] = start[j]; 433 countSet[i][j] = count[j] * stride[j]; 434 strideSet[i][j] = stride[j]; 435 } else { 436 // for the in-track index, it's not so easy... 437 // for first granule, start is what's passed in 438 if (i == 0) { 439 startSet[i][j] = start[j] % vGranuleLength; 440 } else { 441 startSet[i][j] = ((vGranuleLength * granuleNumber) - start[j]) % stride[j]; 442 } 443 // counts may be different for start, end, and middle granules 444 if (i == 0) { 445 // is this the first and only granule? 446 if (granuleSpan == 1) { 447 countSet[i][j] = count[j] * stride[j]; 448 // or is this the first of multiple granules... 449 } else { 450 if (((vGranuleLength * granuleNumber) - start[j]) < (count[j] * stride[j])) { 451 countSet[i][j] = ((vGranuleLength * granuleNumber) - start[j]); 452 } else { 453 countSet[i][j] = count[j] * stride[j]; 454 } 455 countSubtotal += countSet[i][j]; 456 } 457 } else { 458 // middle granules 459 if (i < (granuleSpan - 1)) { 460 countSet[i][j] = vGranuleLength; 461 countSubtotal += countSet[i][j]; 462 } else { 463 // the end granule 464 countSet[i][j] = (count[j] * stride[j]) - countSubtotal; 465 } 466 } 467 // luckily, stride never changes 468 strideSet[i][j] = stride[j]; 469 } 470 } 471 } 472 473 int totalLength = 0; 474 int rangeListCount = 0; 475 ArrayList<Array> arrayList = new ArrayList<Array>(); 476 for (int granuleIdx = 0; granuleIdx < granuleCount; granuleIdx++) { 477 if ((granuleIdx >= loGranuleId) && (granuleIdx <= hiGranuleId)) { 478 Variable var = varMapList.get(loGranuleId + (granuleIdx-loGranuleId)).get(array_name); 479 480 if (var instanceof Structure) { 481 // what to do here? 482 } else { 483 ArrayList<Range> rangeList = new ArrayList<Range>(); 484 for (int dimensionIdx = 0; dimensionIdx < dimensionCount; dimensionIdx++) { 485 logger.trace("Creating new Range: " + startSet[rangeListCount][dimensionIdx] + 486 ", " + (startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1) + ", " + strideSet[rangeListCount][dimensionIdx]); 487 Range range = new Range( 488 startSet[rangeListCount][dimensionIdx], 489 startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1, 490 strideSet[rangeListCount][dimensionIdx] 491 ); 492 rangeList.add(dimensionIdx, range); 493 } 494 rangeListCount++; 495 Array subarray = var.read(rangeList); 496 //dataType = subarray.getElementType(); 497 totalLength += subarray.getSize(); 498 arrayList.add(subarray); 499 } 500 // put in an empty ArrayList placeholder to retain a slot for each granule 501 } else { 502 Array emptyArray = null; 503 arrayList.add(emptyArray); 504 } 505 } 506 507 // last, concatenate the individual NetCDF arrays pulled out 508 509 Class outType; 510 Class arrayType = getArrayType(array_name); 511 RangeProcessor rngProcessor = varToRangeProcessor.get(array_name); 512 if (rngProcessor == null) { 513 outType = getArrayType(array_name); 514 } 515 else { 516 outType = java.lang.Float.TYPE; 517 } 518 Object o = java.lang.reflect.Array.newInstance(outType, totalLength); 519 520 int destPos = 0; 521 int granIdx = 0; 522 523 for (Array a : arrayList) { 524 if (a != null) { 525 Object primArray = a.copyTo1DJavaArray(); 526 primArray = processArray(array_name, arrayType, granIdx, primArray, rngProcessor); 527 System.arraycopy(primArray, 0, o, destPos, (int) a.getSize()); 528 destPos += a.getSize(); 529 } 530 granIdx++; 531 } 532 533 534 return o; 535 } 536 537 public HashMap getVarMap() { 538 return varMapList.get(0); 539 } 540 541 public ArrayList<NetCDFFile> getReaders() { 542 return this.ncdfal; 543 } 544 545 /* pass individual granule pieces just read from dataset through the RangeProcessor */ 546 private Object processArray(String array_name, Class arrayType, int granIdx, Object values, RangeProcessor rngProcessor) { 547 if (rngProcessor == null) { 548 return values; 549 } 550 else { 551 ((AggregationRangeProcessor)rngProcessor).setIndex(granIdx); 552 553 Object outArray = null; 554 if (arrayType == Short.TYPE) { 555 outArray = rngProcessor.processRange((short[])values, null); 556 } else if (arrayType == Byte.TYPE) { 557 outArray = rngProcessor.processRange((byte[])values, null); 558 } else if (arrayType == Float.TYPE) { 559 outArray = rngProcessor.processRange((float[])values, null); 560 } else if (arrayType == Double.TYPE) { 561 outArray = rngProcessor.processRange((double[])values, null); 562 } 563 return outArray; 564 } 565 } 566 567 /* Application can supply a RangeProcessor for an variable 'arrayName' */ 568 public void addRangeProcessor(String arrayName, RangeProcessor rangeProcessor) { 569 varToRangeProcessor.put(arrayName, rangeProcessor); 570 } 571}