001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2015 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see http://www.gnu.org/licenses. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.hydra; 030 031import java.util.ArrayList; 032import java.util.HashMap; 033import java.util.Iterator; 034import java.util.LinkedHashSet; 035import java.util.List; 036 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040import edu.wisc.ssec.mcidasv.data.QualityFlag; 041 042import ucar.ma2.Array; 043import ucar.ma2.DataType; 044import ucar.ma2.Index; 045import ucar.ma2.IndexIterator; 046import ucar.ma2.Range; 047 048import ucar.nc2.Attribute; 049import ucar.nc2.Dimension; 050import ucar.nc2.NetcdfFile; 051import ucar.nc2.Structure; 052import ucar.nc2.Variable; 053 054/** 055 * Provides a view and operations on a set of contiguous data granules as if they 056 * were a single granule. 057 * 058 * This file needs to implement the same signatures NetCDFFile does, 059 * but for aggregations of consecutive granules. 060 * 061 * @author tommyj 062 * 063 */ 064 065public class GranuleAggregation implements MultiDimensionReader { 066 067 private static final Logger logger = LoggerFactory.getLogger(GranuleAggregation.class); 068 069 // this structure holds the NcML readers that get passed in 070 ArrayList<NetcdfFile> nclist = new ArrayList<NetcdfFile>(); 071 072 // this holds the MultiDimensionReaders, here NetCDFFile 073 ArrayList<NetCDFFile> ncdfal = null; 074 075 // need an ArrayList for each variable hashmap structure 076 ArrayList<HashMap<String, Variable>> varMapList = new ArrayList<HashMap<String, Variable>>(); 077 ArrayList<HashMap<String, String[]>> varDimNamesList = new ArrayList<HashMap<String, String[]>>(); 078 ArrayList<HashMap<String, Class>> varDataTypeList = new ArrayList<HashMap<String, Class>>(); 079 080 // map of granule index and granule in-track length for each variable 081 HashMap<String, HashMap<Integer, Integer>> varGranInTrackLengths = new HashMap<String, HashMap<Integer, Integer>>(); 082 HashMap<String, int[]> varAggrDimLengths = new HashMap<String, int[]>(); 083 084 // this object is used to handle granules like VIIRS Imagery EDRs, where scan 085 // gaps of varying sizes and locations in the granule must be removed. If 086 // present, an initial read with these "cut" ranges will be done before subsetting 087 HashMap<Integer, ArrayList<Range>> granCutRanges = new HashMap<Integer, ArrayList<Range>>(); 088 HashMap<Integer, Integer> granCutScans = new HashMap<Integer, Integer>(); 089 090 // except quality flags - only need one hashmap per aggregation 091 // it maps the broken out variable name back to the original packed variable name 092 HashMap<String, QualityFlag> qfMap = null; 093 094 // variable can have bulk array processor set by the application 095 HashMap<String, RangeProcessor> varToRangeProcessor = new HashMap<String, RangeProcessor>(); 096 097 private int granuleCount = -1; 098 private String inTrackDimensionName = null; 099 private String inTrackGeoDimensionName = null; 100 private String crossTrackDimensionName = null; 101 private LinkedHashSet<String> products; 102 private String origName = null; 103 private boolean isEDR = false; 104 105 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, LinkedHashSet<String> products, 106 String inTrackDimensionName, String inTrackGeoDimensionName, 107 String crossTrackDimensionName, boolean isEDR) throws Exception { 108 if (ncdfal == null) throw new Exception("No data: empty Suomi NPP aggregation object"); 109 this.inTrackDimensionName = inTrackDimensionName; 110 this.crossTrackDimensionName = crossTrackDimensionName; 111 this.inTrackGeoDimensionName = inTrackGeoDimensionName; 112 this.ncdfal = ncdfal; 113 this.products = products; 114 this.isEDR = isEDR; 115 init(ncdfal); 116 } 117 118 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, LinkedHashSet<String> products, 119 String inTrackDimensionName, String inTrackGeoDimensionName, 120 String crossTrackDimensionName) throws Exception { 121 this(ncdfal, products, inTrackDimensionName, inTrackGeoDimensionName, crossTrackDimensionName, false); 122 } 123 124 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, LinkedHashSet<String> products, 125 String inTrackDimensionName, String crossTrackDimensionName) throws Exception { 126 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, false); 127 } 128 129 public GranuleAggregation(ArrayList<NetCDFFile> ncdfal, LinkedHashSet<String> products, 130 String inTrackDimensionName, String crossTrackDimensionName, boolean isEDR) throws Exception { 131 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, isEDR); 132 } 133 134 public Class getArrayType(String array_name) { 135 array_name = mapNameIfQualityFlag(array_name); 136 return varDataTypeList.get(0).get(array_name); 137 } 138 139 public String[] getDimensionNames(String array_name) { 140 array_name = mapNameIfQualityFlag(array_name); 141 return varDimNamesList.get(0).get(array_name); 142 } 143 144 public int[] getDimensionLengths(String array_name) { 145 array_name = mapNameIfQualityFlag(array_name); 146 return varAggrDimLengths.get(array_name); 147 } 148 149 private String mapNameIfQualityFlag(String array_name) { 150 // only applies if name is from a packed quality flag 151 // we pull data from the "mapped" variable name, a packed byte 152 if (qfMap != null) { 153 if (qfMap.containsKey(array_name)) { 154 origName = array_name; 155 QualityFlag qf = qfMap.get(array_name); 156 String mappedName = qf.getPackedName(); 157 logger.debug("Key: " + array_name + " mapped to: " + mappedName); 158 return mappedName; 159 } 160 } 161 return array_name; 162 } 163 164 /** 165 * @return the isEDR 166 */ 167 public boolean isEDR() { 168 return isEDR; 169 } 170 171 /** 172 * @param isEDR the isEDR to set 173 */ 174 public void setEDR(boolean isEDR) { 175 this.isEDR = isEDR; 176 } 177 178 public float[] getFloatArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 179 return (float[]) readArray(array_name, start, count, stride); 180 } 181 182 public int[] getIntArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 183 return (int[]) readArray(array_name, start, count, stride); 184 } 185 186 public double[] getDoubleArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 187 return (double[]) readArray(array_name, start, count, stride); 188 } 189 190 public short[] getShortArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 191 return (short[]) readArray(array_name, start, count, stride); 192 } 193 194 public byte[] getByteArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 195 return (byte[]) readArray(array_name, start, count, stride); 196 } 197 198 public Object getArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 199 return readArray(array_name, start, count, stride); 200 } 201 202 public HDFArray getGlobalAttribute(String attr_name) throws Exception { 203 throw new Exception("GranuleAggregation.getGlobalAttributes: Unimplemented"); 204 } 205 206 public HDFArray getArrayAttribute(String array_name, String attr_name) throws Exception { 207 Variable var = varMapList.get(0).get(array_name); 208 if (var == null) return null; 209 210 Attribute attr = var.findAttribute(attr_name); 211 if (attr == null) return null; 212 213 Array attrVals = attr.getValues(); 214 DataType dataType = attr.getDataType(); 215 Object array = attrVals.copyTo1DJavaArray(); 216 217 HDFArray harray = null; 218 219 if (dataType.getPrimitiveClassType() == Float.TYPE) { 220 harray = HDFArray.make((float[])array); 221 } 222 else if (dataType.getPrimitiveClassType() == Double.TYPE) { 223 harray = HDFArray.make((double[])array); 224 } 225 else if (dataType == DataType.STRING) { 226 harray = HDFArray.make((String[])array); 227 } 228 else if (dataType.getPrimitiveClassType() == Short.TYPE) { 229 harray = HDFArray.make((short[])array); 230 } 231 else if (dataType.getPrimitiveClassType() == Integer.TYPE) { 232 harray = HDFArray.make((int[])array); 233 } 234 return harray; 235 } 236 237 public void close() throws Exception { 238 // close each NetCDF file 239 for (NetcdfFile n : nclist) { 240 n.close(); 241 } 242 } 243 244 private void init(ArrayList<NetCDFFile> ncdfal) throws Exception { 245 246 logger.debug("init in..."); 247 // make a NetCDFFile object from the NcML for each granule 248 for (NetCDFFile n : ncdfal) { 249 logger.debug("loading another NetCDF file from NcML..."); 250 NetcdfFile ncfile = n.getNetCDFFile(); 251 nclist.add(ncfile); 252 } 253 254 granuleCount = nclist.size(); 255 logger.debug("Granule count: " + granuleCount); 256 257 // All files do NOT have the same structure, so need to look at each ncfile 258 // For ex, some MODIS granules have slightly different in-track and along-track 259 // lengths 260 261 NetcdfFile ncfile = null; 262 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 263 264 // good place to initialize the cut Range ArrayList for each granule 265 Integer granuleIndex = new Integer(ncIdx); 266 ArrayList<Range> al = new ArrayList<Range>(); 267 granCutRanges.put(granuleIndex, al); 268 int cutScanCount = 0; 269 270 ncfile = nclist.get(ncIdx); 271 272 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 273 while (varIter.hasNext()) { 274 Variable var = varIter.next(); 275 logger.trace("Variable " + var.getShortName() + ", Rank: " + var.getRank()); 276 varAggrDimLengths.put(var.getFullName(), new int[var.getRank()]); 277 varGranInTrackLengths.put(var.getFullName(), new HashMap<Integer, Integer>()); 278 279 // Here, let's try to check the data for EDR fill lines 280 // and if found, try to handle it by simply adjusting the dimensions 281 // for this granule. Sound like a plan? We'll see... 282 283 if (isEDR) { 284 285 // look through lat grid, look for missing scans 286 String varName = var.getShortName(); 287 if ((varName.endsWith("Latitude")) || (varName.endsWith("Latitude_TC"))){ 288 // iterate through the scan lines, looking for fill lines 289 // NOTE: we only need to check the first column! so set 290 // up an appropriate Range to cut the read down significantly 291 int[] shape = var.getShape(); 292 ArrayList<Range> alr = new ArrayList<Range>(); 293 alr.add(new Range(0, shape[0] - 1, 1)); 294 alr.add(new Range(0, 1, 1)); 295 Array a = var.read(alr); 296 int scanLength = shape[1]; 297 Index index = a.getIndex(); 298 float fVal = 0.0f; 299 300 int rangeOffset = 1; 301 int rangeCount = 0; 302 boolean prvScanWasCut = false; 303 boolean needClosingRange = false; 304 boolean hadCutRanges = false; 305 boolean someMissing = false; 306 307 for (int i = 0; i < shape[0]; i++) { 308 309 someMissing = false; 310 fVal = a.getFloat(index.set(i, 0)); 311 if (fVal < -90.0f) { 312 someMissing = true; 313 } 314 315 if (someMissing) { 316 hadCutRanges = true; 317 cutScanCount++; 318 logger.trace("Found a cut scan " + (i + 1) 319 + ", last val: " + fVal); 320 if ((prvScanWasCut) || (i == 0)) { 321 if (i == 0) { 322 rangeOffset = 1; 323 } else { 324 rangeOffset = i + 2; 325 } 326 } else { 327 try { 328 // We are using 2D ranges 329 logger.trace("Adding Range: " + rangeOffset 330 + ", " + i + ", 1"); 331 al.add(new Range(rangeOffset, i, 1)); 332 logger.trace("Adding Range: " + 1 + ", " 333 + (scanLength - 1) + ", 1"); 334 al.add(new Range(0, scanLength - 1, 1)); 335 } catch (Exception e) { 336 e.printStackTrace(); 337 } 338 rangeCount = 0; 339 rangeOffset = i + 1; 340 } 341 prvScanWasCut = true; 342 } else { 343 prvScanWasCut = false; 344 rangeCount += scanLength; 345 } 346 347 // check to see if closing Range needed, good data at end 348 if ((! prvScanWasCut) && (i == (scanLength - 1))) { 349 needClosingRange = true; 350 } 351 } 352 353 if (needClosingRange) { 354 // We are using 2D ranges 355 al.add(new Range(rangeOffset, rangeOffset + shape[0] 356 - 1, 1)); 357 al.add(new Range(0, scanLength - 1, 1)); 358 logger.trace("Adding closing cut Range, offs: " 359 + rangeOffset + ", len: " + rangeCount); 360 } 361 362 // if only one contiguous range, process as a normal clean granule 363 if (! hadCutRanges) { 364 al.clear(); 365 } 366 367 granCutScans.put(granuleIndex, new Integer(cutScanCount)); 368 logger.debug("Total scans cut this granule: " 369 + cutScanCount); 370 371 } 372 } else { 373 granCutScans.put(granuleIndex, new Integer(0)); 374 } 375 } 376 } 377 378 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 379 380 ncfile = nclist.get(ncIdx); 381 382 HashMap<String, Variable> varMap = new HashMap<String, Variable>(); 383 HashMap<String, String[]> varDimNames = new HashMap<String, String[]>(); 384 HashMap<String, Class> varDataType = new HashMap<String, Class>(); 385 386 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 387 int varInTrackIndex = -1; 388 while (varIter.hasNext()) { 389 Variable var = (Variable) varIter.next(); 390 391 boolean foundProduct = false; 392 for (String s : products) { 393 if (s.contains(var.getFullName())) { 394 logger.trace("Valid product: " + var.getFullName()); 395 foundProduct = true; 396 break; 397 } 398 } 399 400 if (! foundProduct) { 401 logger.trace("Skipping variable: " + var.getFullName()); 402 continue; 403 } 404 405 if (var instanceof Structure) { 406 // simply skip these, applicable only to IASI far as I know 407 continue; 408 } 409 410 int rank = var.getRank(); 411 412 // bypass any less-than-2D variables for now... 413 if (rank < 2) { 414 continue; 415 } 416 417 String varName = var.getFullName(); 418 varMap.put(varName, var); 419 Iterator<Dimension> dimIter = var.getDimensions().iterator(); 420 String[] dimNames = new String[rank]; 421 int[] dimLengths = new int[rank]; 422 int cnt = 0; 423 boolean notDisplayable = false; 424 varInTrackIndex = getInTrackIndex(var); 425 426 while (dimIter.hasNext()) { 427 Dimension dim = dimIter.next(); 428 String s = dim.getShortName(); 429 if ((s != null) && (!s.isEmpty())) { 430 if ((! s.equals(inTrackDimensionName)) && 431 ((! s.startsWith("Band")) && (cnt == 0)) && 432 (! varName.endsWith("Latitude")) && 433 (! varName.endsWith("Latitude_TC")) && 434 (! varName.endsWith("Longitude")) && 435 (! varName.endsWith("Longitude_TC")) && 436 (! s.equals(crossTrackDimensionName))) { 437 notDisplayable = true; 438 break; 439 } 440 } 441 String dimName = dim.getShortName(); 442 logger.debug("GranuleAggregation init, variable: " + varName + ", dimension name: " + dimName + ", length: " + dim.getLength()); 443 if (dimName == null) dimName = "dim" + cnt; 444 if (dimName.isEmpty()) { 445 dimName = "dim" + cnt; 446 } 447 dimNames[cnt] = dimName; 448 dimLengths[cnt] = dim.getLength(); 449 cnt++; 450 } 451 452 // skip to next variable if it's not displayable data 453 if (notDisplayable) continue; 454 455 // adjust in-track dimension if needed (scans were cut) 456 int cutScans = granCutScans.get(ncIdx); 457 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] - cutScans; 458 459 // XXX TJJ - can below block go away? Think so... 460 int[] aggrDimLengths = varAggrDimLengths.get(varName); 461 for (int i = 0; i < rank; i++) { 462 if (i == varInTrackIndex) { 463 aggrDimLengths[i] += dimLengths[i]; 464 } else { 465 aggrDimLengths[i] = dimLengths[i]; 466 } 467 } 468 469 varDimNames.put(varName, dimNames); 470 varDataType.put(varName, var.getDataType().getPrimitiveClassType()); 471 472 if (varInTrackIndex < 0) { 473 logger.debug("Skipping variable with unknown dimension: " + var.getFullName()); 474 continue; 475 } 476 477 HashMap<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(varName); 478 granIdxToInTrackLen.put(ncIdx, new Integer(dimLengths[varInTrackIndex])); 479 480 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] * granuleCount; 481 varDataType.put(varName, var.getDataType().getPrimitiveClassType()); 482 } 483 484 // add the new hashmaps to our enclosing lists 485 varMapList.add(varMap); 486 varDimNamesList.add(varDimNames); 487 varDataTypeList.add(varDataType); 488 489 } 490 } 491 492 /** 493 * Based on the names of the variable dimensions, determine the in-track index. 494 * 495 * @param v {@code Variable} that {@literal "contains"} dimension names that 496 * allow for inference of the in-track index. {@code null} is allowed. 497 * 498 * @return correct index (0 or greater), or -1 if error. 499 */ 500 501 private int getInTrackIndex(Variable v) { 502 503 int index = -1; 504 boolean is2D = false; 505 boolean is3D = false; 506 507 String inTrackName = null; 508 509 // typical sanity check 510 if (v == null) return index; 511 512 // lat/lon vars have different dimension names 513 if ((v.getFullName().endsWith("Latitude")) || 514 (v.getFullName().endsWith("Latitude_TC")) || 515 (v.getFullName().endsWith("Longitude")) || 516 (v.getFullName().endsWith("LongitudeTC"))) { 517 if (v.getFullName().startsWith("All_Data")) { 518 inTrackName = inTrackDimensionName; 519 } else { 520 inTrackName = inTrackGeoDimensionName; 521 } 522 } else { 523 inTrackName = inTrackDimensionName; 524 } 525 // pull out the dimensions 526 List<Dimension> dList = v.getDimensions(); 527 528 // right now, we only handle 2D and 3D variables. 529 // TJJ XXX it does get trickier, and we will have to expand this 530 // to deal with for example CrIS data... 531 int numDimensions = dList.size(); 532 533 // the only 4D data right now is CrIS, return 0 534 if (numDimensions == 4) return 0; 535 536 if ((numDimensions == 2) || (numDimensions == 3)) { 537 if (numDimensions == 2) is2D = true; 538 if (numDimensions == 3) is3D = true; 539 } else { 540 return index; 541 } 542 543 // if the data is 2D, we use the SwathAdapter class, 544 // if 3D, we use the SpectrumAdapter class 545 for (int i = 0; i < numDimensions; i++) { 546 if (is2D) { 547 // XXX TJJ - if empty name, in-track index is 0 548 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 549 logger.trace("Empty dimension name!, assuming in-track dim is 0"); 550 return 0; 551 } 552 if (dList.get(i).getShortName().equals(inTrackName)) { 553 index = i; 554 break; 555 } 556 } 557 if (is3D) { 558 // XXX TJJ - if empty name, in-track index is 0 559 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 560 logger.debug("Empty dimension name!, assuming in-track dim is 0"); 561 return 0; 562 } 563 if (dList.get(i).getShortName().equals(inTrackName)) { 564 index = i; 565 break; 566 } 567 } 568 } 569 570 // hopefully we found the right one 571 return index; 572 } 573 574 private synchronized Object readArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 575 576 array_name = mapNameIfQualityFlag(array_name); 577 // how many dimensions are we dealing with 578 int dimensionCount = start.length; 579 580 // pull out a representative variable so we can determine which index is in-track 581 Variable vTmp = varMapList.get(0).get(array_name); 582 int vInTrackIndex = getInTrackIndex(vTmp); 583 584 int loGranuleId = 0; 585 int hiGranuleId = 0; 586 587 HashMap<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(array_name); 588 int numGrans = granIdxToInTrackLen.size(); 589 590 int[] vGranuleLengths = new int[numGrans]; 591 for (int k = 0; k < numGrans; k++) { 592 vGranuleLengths[k] = granIdxToInTrackLen.get(k); 593 logger.debug("readArray, gran len: " + vGranuleLengths[k] + ", scans cut: " + granCutScans.get(k)); 594 } 595 596 int strt = start[vInTrackIndex]; 597 int stp = strt + (count[vInTrackIndex] - 1) * stride[vInTrackIndex]; 598 int cnt = 0; 599 for (int k = 0; k < numGrans; k++) { 600 int granLen = granIdxToInTrackLen.get(k); 601 cnt += granLen; 602 if (strt < cnt) { 603 loGranuleId = k; 604 break; 605 } 606 } 607 608 cnt = 0; 609 for (int k = 0; k < numGrans; k++) { 610 int granLen = granIdxToInTrackLen.get(k); 611 cnt += granLen; 612 if (stp < cnt) { 613 hiGranuleId = k; 614 break; 615 } 616 } 617 618 // next, we break out the offsets, counts, and strides for each granule 619 int granuleSpan = hiGranuleId - loGranuleId + 1; 620 621 logger.debug("readArray req, loGran: " + loGranuleId + ", hiGran: " + 622 hiGranuleId + ", granule span: " + granuleSpan + ", dimCount: " + dimensionCount); 623 624 for (int i = 0; i < dimensionCount; i++) { 625 logger.debug("start[" + i + "]: " + start[i]); 626 logger.debug("count[" + i + "]: " + count[i]); 627 logger.debug("stride[" + i + "]: " + stride[i]); 628 } 629 630 int [][] startSet = new int [granuleSpan][dimensionCount]; 631 int [][] countSet = new int [granuleSpan][dimensionCount]; 632 int [][] strideSet = new int [granuleSpan][dimensionCount]; 633 int countSubtotal = 0; 634 635 int inTrackTotal = 0; 636 for (int i = 0; i < loGranuleId; i++) { 637 inTrackTotal += vGranuleLengths[i]; 638 } 639 640 // this part is a little tricky - set the values for each granule we need to access for this read 641 for (int i = 0; i < granuleSpan; i++) { 642 inTrackTotal += vGranuleLengths[loGranuleId+i]; 643 for (int j = 0; j < dimensionCount; j++) { 644 // for all indeces other than the in-track index, the numbers match what was passed in 645 if (j != vInTrackIndex) { 646 startSet[i][j] = start[j]; 647 countSet[i][j] = count[j] * stride[j]; 648 strideSet[i][j] = stride[j]; 649 } else { 650 // for the in-track index, it's not so easy... 651 // for first granule, start is what's passed in 652 if (i == 0) { 653 startSet[i][j] = start[j] - (inTrackTotal - vGranuleLengths[loGranuleId]); 654 } else { 655 startSet[i][j] = (inTrackTotal - start[j]) % stride[j]; 656 // TJJ Sep 2013, zero-base starts that offset into subsequent granules 657 if (startSet[i][j] > 0) { 658 startSet[i][j]--; 659 } 660 } 661 // counts may be different for start, end, and middle granules 662 if (i == 0) { 663 // is this the first and only granule? 664 if (granuleSpan == 1) { 665 countSet[i][j] = count[j] * stride[j]; 666 // or is this the first of multiple granules... 667 } else { 668 if ((inTrackTotal - start[j]) < (count[j] * stride[j])) { 669 countSet[i][j] = inTrackTotal - start[j]; 670 } else { 671 countSet[i][j] = count[j] * stride[j]; 672 } 673 countSubtotal += countSet[i][j]; 674 } 675 } else { 676 // middle granules 677 if (i < (granuleSpan - 1)) { 678 countSet[i][j] = vGranuleLengths[loGranuleId+i]; 679 countSubtotal += countSet[i][j]; 680 } else { 681 // the end granule 682 countSet[i][j] = (count[j] * stride[j]) - countSubtotal; 683 // XXX TJJ - limiting count to valid numbers here, why?? 684 // need to revisit, see why this condition manifests 685 if (countSet[i][j] > (vGranuleLengths[loGranuleId+i] - startSet[i][j])) 686 countSet[i][j] = vGranuleLengths[loGranuleId+i] - startSet[i][j]; 687 } 688 } 689 // luckily, stride never changes 690 strideSet[i][j] = stride[j]; 691 } 692 } 693 } 694 695 int totalLength = 0; 696 int rangeListCount = 0; 697 ArrayList<Array> arrayList = new ArrayList<Array>(); 698 for (int granuleIdx = 0; granuleIdx < granuleCount; granuleIdx++) { 699 if ((granuleIdx >= loGranuleId) && (granuleIdx <= hiGranuleId)) { 700 Variable var = varMapList.get(loGranuleId + (granuleIdx-loGranuleId)).get(array_name); 701 702 if (var instanceof Structure) { 703 // what to do here? 704 } else { 705 ArrayList<Range> rangeList = new ArrayList<Range>(); 706 for (int dimensionIdx = 0; dimensionIdx < dimensionCount; dimensionIdx++) { 707 logger.debug("Creating new Range: " + startSet[rangeListCount][dimensionIdx] + 708 ", " + (startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1) + ", " + strideSet[rangeListCount][dimensionIdx]); 709 Range range = new Range( 710 startSet[rangeListCount][dimensionIdx], 711 startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1, 712 strideSet[rangeListCount][dimensionIdx] 713 ); 714 rangeList.add(dimensionIdx, range); 715 } 716 rangeListCount++; 717 718 // If there were chunks of fill data to remove... 719 ArrayList<Range> al = granCutRanges.get(new Integer(granuleIdx)); 720 if (! al.isEmpty()) { 721 ArrayList<Variable> varChunks = new ArrayList<Variable>(); 722 for (int rangeCount = 0; rangeCount < al.size(); rangeCount+=2) { 723 ArrayList<Range> rl = new ArrayList<Range>(); 724 rl.add(al.get(rangeCount)); 725 rl.add(al.get(rangeCount + 1)); 726 varChunks.add(var.section(rl)); 727 } 728 729 int [] newShape = var.getShape(); 730 int cutScans = granCutScans.get(granuleIdx); 731 newShape[0] = newShape[0] - cutScans; 732 logger.trace("New Shape: " + newShape[0] + ", " + newShape[1]); 733 Array single = Array.factory(var.getDataType(), newShape); 734 735 // now read variable chunk data into single contiguous array 736 int idx = 0; 737 for (Variable v : varChunks) { 738 Array data = v.read(); 739 int [] tmpShape = v.getShape(); 740 for (int tIdx = 0; tIdx < tmpShape.length; tIdx++) { 741 logger.trace("Shape[" + tIdx + "]: " + tmpShape[tIdx]); 742 } 743 IndexIterator ii = data.getIndexIterator(); 744 while (ii.hasNext()) { 745 single.setFloat(idx, ii.getFloatNext()); 746 idx++; 747 } 748 } 749 750 // finally, apply subset ranges 751 Array subarray = single.section(rangeList); 752 totalLength += subarray.getSize(); 753 arrayList.add(subarray); 754 logger.debug("Size of final data array: " + subarray.getSize()); 755 756 } else { 757 Array subarray = var.read(rangeList); 758 totalLength += subarray.getSize(); 759 arrayList.add(subarray); 760 } 761 762 } 763 // put in an empty ArrayList placeholder to retain a slot for each granule 764 } else { 765 Array emptyArray = null; 766 arrayList.add(emptyArray); 767 } 768 } 769 770 // last, concatenate the individual NetCDF arrays pulled out 771 772 Class outType; 773 Class arrayType = getArrayType(array_name); 774 RangeProcessor rngProcessor = varToRangeProcessor.get(array_name); 775 if (rngProcessor == null) { 776 outType = getArrayType(array_name); 777 } 778 else { 779 outType = java.lang.Float.TYPE; 780 } 781 Object o = java.lang.reflect.Array.newInstance(outType, totalLength); 782 783 int destPos = 0; 784 int granIdx = 0; 785 786 for (Array a : arrayList) { 787 if (a != null) { 788 Object primArray = a.copyTo1DJavaArray(); 789 primArray = processArray(array_name, arrayType, granIdx, primArray, rngProcessor, start, count); 790 System.arraycopy(primArray, 0, o, destPos, (int) a.getSize()); 791 destPos += a.getSize(); 792 } 793 granIdx++; 794 } 795 796 return o; 797 } 798 799 /** 800 * @param qfMap the qfMap to set 801 */ 802 public void setQfMap(HashMap<String, QualityFlag> qfMap) { 803 this.qfMap = qfMap; 804 } 805 806 public HashMap getVarMap() { 807 return varMapList.get(0); 808 } 809 810 public ArrayList<NetCDFFile> getReaders() { 811 return this.ncdfal; 812 } 813 814 /* pass individual granule pieces just read from dataset through the RangeProcessor */ 815 private Object processArray(String array_name, Class arrayType, int granIdx, Object values, RangeProcessor rngProcessor, int[] start, int[] count) { 816 817 if (rngProcessor == null) { 818 return values; 819 } 820 else { 821 ((AggregationRangeProcessor)rngProcessor).setWhichRangeProcessor(granIdx); 822 823 boolean processAlongMultiScaleDim = false; 824 825 if (rngProcessor.hasMultiDimensionScale()) { // this data variable has an array > 1 of scale/offsets. For example, one for each band. 826 rngProcessor.setMultiScaleIndex(start[rngProcessor.getMultiScaleDimensionIndex()]); 827 if (count[rngProcessor.getMultiScaleDimensionIndex()] > 1) { // if the multiScaleDim is > 1, use processAlongMultiScaleDim below 828 processAlongMultiScaleDim = true; 829 } 830 } 831 832 Object outArray = null; 833 834 if (processAlongMultiScaleDim) { 835 836 if (arrayType == Short.TYPE) { 837 outArray = rngProcessor.processAlongMultiScaleDim((short[])values); 838 } else if (arrayType == Byte.TYPE) { 839 outArray = rngProcessor.processAlongMultiScaleDim((byte[])values); 840 } else if (arrayType == Float.TYPE) { 841 outArray = values; 842 } else if (arrayType == Double.TYPE) { 843 outArray = values; 844 } 845 846 } 847 else { 848 849 if (arrayType == Short.TYPE) { 850 outArray = rngProcessor.processRange((short[]) values, null); 851 } else if (arrayType == Byte.TYPE) { 852 // if variable is a bit-field quality flag, apply mask 853 if (qfMap.containsKey(origName)) { 854 QualityFlag qf = qfMap.get(origName); 855 outArray = rngProcessor.processRangeQualityFlag((byte[]) values, null, qf); 856 } else { 857 outArray = rngProcessor.processRange((byte[]) values, null); 858 } 859 } else if (arrayType == Float.TYPE) { 860 outArray = rngProcessor.processRange((float[]) values, null); 861 } else if (arrayType == Double.TYPE) { 862 outArray = rngProcessor.processRange((double[]) values, null); 863 } 864 865 } 866 867 return outArray; 868 } 869 } 870 871 /* Application can supply a RangeProcessor for a variable 'arrayName' */ 872 public void addRangeProcessor(String arrayName, RangeProcessor rangeProcessor) { 873 varToRangeProcessor.put(arrayName, rangeProcessor); 874 } 875 876}