001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2024 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas/ 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see https://www.gnu.org/licenses/. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.hydra; 030 031import java.util.ArrayList; 032import java.util.HashMap; 033import java.util.Iterator; 034import java.util.List; 035import java.util.Map; 036import java.util.Set; 037 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041import edu.wisc.ssec.mcidasv.data.QualityFlag; 042 043import ucar.ma2.Array; 044import ucar.ma2.DataType; 045import ucar.ma2.Index; 046import ucar.ma2.IndexIterator; 047import ucar.ma2.Range; 048import ucar.nc2.Attribute; 049import ucar.nc2.Dimension; 050import ucar.nc2.NetcdfFile; 051import ucar.nc2.Structure; 052import ucar.nc2.Variable; 053 054/** 055 * Provides a view and operations on a set of contiguous data granules as if they 056 * were a single granule. 057 * 058 * This file needs to implement the same signatures NetCDFFile does, 059 * but for aggregations of consecutive granules. 060 * 061 * @author tommyj 062 * 063 */ 064 065public class GranuleAggregation implements MultiDimensionReader { 066 067 private static final Logger logger = 068 LoggerFactory.getLogger(GranuleAggregation.class); 069 070 // this structure holds the NcML readers that get passed in 071 List<NetcdfFile> nclist = new ArrayList<>(); 072 073 // this holds the MultiDimensionReaders, here NetCDFFile 074 List<NetCDFFile> ncdfal = null; 075 076 // need an ArrayList for each variable hashmap structure 077 List<Map<String, Variable>> varMapList = new ArrayList<>(); 078 List<Map<String, String[]>> varDimNamesList = new ArrayList<>(); 079 List<Map<String, DataType>> varDataTypeList = new ArrayList<>(); 080 081 // map of granule index and granule in-track length for each variable 082 Map<String, Map<Integer, Integer>> varGranInTrackLengths = new HashMap<>(); 083 Map<String, int[]> varAggrDimLengths = new HashMap<>(); 084 085 // this object is used to handle granules like VIIRS Imagery EDRs, where scan 086 // gaps of varying sizes and locations in the granule must be removed. If 087 // present, an initial read with these "cut" ranges will be done before subsetting 088 Map<Integer, List<Range>> granCutRanges = new HashMap<>(); 089 Map<Integer, Integer> granCutScans = new HashMap<>(); 090 091 // except quality flags - only need one hashmap per aggregation 092 // it maps the broken out variable name back to the original packed variable name 093 Map<String, QualityFlag> qfMap = null; 094 095 // For those variables which are assembled from other variables based on LUTs 096 Map<String, float[]> lutMap = null; 097 098 // variable can have bulk array processor set by the application 099 Map<String, RangeProcessor> varToRangeProcessor = new HashMap<>(); 100 101 private int granuleCount = -1; 102 private String inTrackDimensionName = null; 103 private String inTrackGeoDimensionName = null; 104 private String crossTrackDimensionName = null; 105 private Set<String> products; 106 private String origName = ""; 107 // assume we are working with VIIRS, will toggle if not 108 private boolean isVIIRS = true; 109 110 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 111 String inTrackDimensionName, String inTrackGeoDimensionName, 112 String crossTrackDimensionName, boolean isVIIRS) throws Exception { 113 if (ncdfal == null) throw new Exception("No data: empty Suomi NPP aggregation object"); 114 this.inTrackDimensionName = inTrackDimensionName; 115 this.crossTrackDimensionName = crossTrackDimensionName; 116 this.inTrackGeoDimensionName = inTrackGeoDimensionName; 117 this.ncdfal = ncdfal; 118 this.products = products; 119 this.isVIIRS = isVIIRS; 120 init(ncdfal); 121 } 122 123 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 124 String inTrackDimensionName, String inTrackGeoDimensionName, 125 String crossTrackDimensionName) throws Exception { 126 this(ncdfal, products, inTrackDimensionName, inTrackGeoDimensionName, crossTrackDimensionName, false); 127 } 128 129 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 130 String inTrackDimensionName, String crossTrackDimensionName) throws Exception { 131 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, false); 132 } 133 134 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 135 String inTrackDimensionName, String crossTrackDimensionName, boolean isEDR) throws Exception { 136 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, isEDR); 137 } 138 139 public DataType getArrayType(String array_name) { 140 array_name = mapNameIfQualityFlag(array_name); 141 array_name = mapNameIfLUTVar(array_name); 142 return varDataTypeList.get(0).get(array_name); 143 } 144 145 public String[] getDimensionNames(String array_name) { 146 array_name = mapNameIfQualityFlag(array_name); 147 array_name = mapNameIfLUTVar(array_name); 148 return varDimNamesList.get(0).get(array_name); 149 } 150 151 public int[] getDimensionLengths(String array_name) { 152 array_name = mapNameIfQualityFlag(array_name); 153 array_name = mapNameIfLUTVar(array_name); 154 return varAggrDimLengths.get(array_name); 155 } 156 157 private String mapNameIfQualityFlag(String array_name) { 158 // only applies if name is from a packed quality flag 159 // we pull data from the "mapped" variable name, a packed byte 160 if (qfMap != null) { 161 if (qfMap.containsKey(array_name)) { 162 origName = array_name; 163 QualityFlag qf = qfMap.get(array_name); 164 String mappedName = qf.getPackedName(); 165 logger.debug("Key: " + array_name + " mapped to: " + mappedName); 166 return mappedName; 167 } 168 } 169 return array_name; 170 } 171 172 private String mapNameIfLUTVar(String array_name) { 173 // only applies if name is from a LUT pseudo variable 174 // we pull data from a "mapped" variable name, and apply a LUT to that variable 175 176 if (lutMap != null) { 177 if (lutMap.containsKey(array_name)) { 178 origName = array_name; 179 String mappedName = array_name.substring(0, array_name.length() - 3); 180 logger.debug("Key: " + array_name + " mapped to: " + mappedName); 181 return mappedName; 182 } 183 } 184 return array_name; 185 } 186 187 public float[] getFloatArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 188 return (float[]) readArray(array_name, start, count, stride); 189 } 190 191 public int[] getIntArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 192 return (int[]) readArray(array_name, start, count, stride); 193 } 194 195 public double[] getDoubleArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 196 return (double[]) readArray(array_name, start, count, stride); 197 } 198 199 public short[] getShortArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 200 return (short[]) readArray(array_name, start, count, stride); 201 } 202 203 public byte[] getByteArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 204 return (byte[]) readArray(array_name, start, count, stride); 205 } 206 207 public Object getArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 208 return readArray(array_name, start, count, stride); 209 } 210 211 public HDFArray getGlobalAttribute(String attr_name) throws Exception { 212 throw new Exception("GranuleAggregation.getGlobalAttributes: Unimplemented"); 213 } 214 215 public HDFArray getArrayAttribute(String array_name, String attr_name) throws Exception { 216 Variable var = varMapList.get(0).get(array_name); 217 if (var == null) return null; 218 219 Attribute attr = var.findAttribute(attr_name); 220 if (attr == null) return null; 221 222 Array attrVals = attr.getValues(); 223 DataType dataType = attr.getDataType(); 224 Object array = attrVals.copyTo1DJavaArray(); 225 226 HDFArray harray = null; 227 228 if (dataType.getPrimitiveClassType() == Float.TYPE) { 229 harray = HDFArray.make((float[])array); 230 } 231 else if (dataType.getPrimitiveClassType() == Double.TYPE) { 232 harray = HDFArray.make((double[])array); 233 } 234 else if (dataType == DataType.STRING) { 235 harray = HDFArray.make((String[])array); 236 } 237 else if (dataType.getPrimitiveClassType() == Short.TYPE) { 238 harray = HDFArray.make((short[])array); 239 } 240 else if (dataType.getPrimitiveClassType() == Integer.TYPE) { 241 harray = HDFArray.make((int[])array); 242 } 243 return harray; 244 } 245 246 public void close() throws Exception { 247 // close each NetCDF file 248 for (NetcdfFile n : nclist) { 249 n.close(); 250 } 251 } 252 253 private void init(List<NetCDFFile> ncdfal) throws Exception { 254 255 logger.debug("init in..."); 256 // make a NetCDFFile object from the NcML for each granule 257 for (NetCDFFile n : ncdfal) { 258 logger.debug("loading another NetCDF file from NcML..."); 259 NetcdfFile ncfile = n.getNetCDFFile(); 260 nclist.add(ncfile); 261 } 262 263 granuleCount = nclist.size(); 264 logger.debug("Granule count: " + granuleCount); 265 266 // All files do NOT have the same structure, so need to look at each ncfile 267 // For ex, some MODIS granules have slightly different in-track and along-track 268 // lengths 269 270 NetcdfFile ncfile = null; 271 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 272 273 // good place to initialize the cut Range ArrayList for each granule 274 Integer granuleIndex = new Integer(ncIdx); 275 List<Range> al = new ArrayList<>(); 276 277 int cutScanCount = 0; 278 279 ncfile = nclist.get(ncIdx); 280 281 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 282 while (varIter.hasNext()) { 283 Variable var = varIter.next(); 284 logger.trace("Variable " + var.getShortName() + ", Rank: " + var.getRank()); 285 varAggrDimLengths.put(var.getFullName(), new int[var.getRank()]); 286 varGranInTrackLengths.put(var.getFullName(), new HashMap<>()); 287 288 // Here, let's try to check the data for EDR fill lines 289 // and if found, try to handle it by simply adjusting the dimensions 290 // for this granule. Sound like a plan? We'll see... 291 292 // TJJ May 2016 293 // "simply adjusting the dimensions" he says 294 // Anyway, we now do this check for EDRs and SDRs, it can manifest for both 295 296 if (isVIIRS) { 297 298 // look through lat grid, look for missing scans 299 String varName = var.getShortName(); 300 if (varName.endsWith("Latitude")) { 301 // iterate through the scan lines, looking for fill lines 302 // NOTE: we only need to check the first column! so set 303 // up an appropriate Range to cut the read down significantly 304 int[] shape = var.getShape(); 305 List<Range> alr = new ArrayList<>(); 306 alr.add(new Range(0, shape[0] - 1, 1)); 307 alr.add(new Range(0, 1, 1)); 308 Array a = var.read(alr); 309 int granLength = shape[0]; 310 int scanLength = shape[1]; 311 Index index = a.getIndex(); 312 float fVal = 0.0f; 313 314 int rangeOffset = 0; 315 boolean prvScanWasCut = false; 316 boolean needClosingRange = false; 317 boolean hadCutRanges = false; 318 boolean someMissing = false; 319 320 for (int i = 0; i < shape[0]; i++) { 321 322 someMissing = false; 323 fVal = a.getFloat(index.set(i, 0)); 324 if (fVal < -90.0f) { 325 someMissing = true; 326 } 327 328 if (someMissing) { 329 hadCutRanges = true; 330 cutScanCount++; 331 logger.trace("Found a cut scan " + i + ", last val: " + fVal); 332 if ((prvScanWasCut) || (i == 0)) { 333 if (i == 0) { 334 rangeOffset = 0; 335 } else { 336 rangeOffset = i + 1; 337 } 338 } else { 339 try { 340 // We are using 2D ranges 341 logger.trace("Adding Range: " + rangeOffset 342 + ", " + (i - 1) + ", 1"); 343 al.add(new Range(rangeOffset, i - 1, 1)); 344 logger.trace("Adding Range: " + 0 + ", " 345 + (scanLength - 1) + ", 1"); 346 al.add(new Range(0, scanLength - 1, 1)); 347 } catch (Exception e) { 348 logger.trace("problem creating range", e); 349 } 350 rangeOffset = i; 351 } 352 prvScanWasCut = true; 353 } else { 354 prvScanWasCut = false; 355 } 356 357 // check to see if closing Range needed, good data at end 358 if ((! prvScanWasCut) && (i == (granLength - 1))) { 359 if (hadCutRanges) { 360 needClosingRange = true; 361 } 362 } 363 } 364 365 if (needClosingRange) { 366 // We are using 2D ranges 367 logger.trace("Adding closing cut range: " + rangeOffset + ", " + (shape[0] - 1) + ", 1"); 368 al.add(new Range(rangeOffset, shape[0] - 1, 1)); 369 al.add(new Range(0, scanLength - 1, 1)); 370 } 371 372 // if only one contiguous range, process as a normal clean granule 373 if (! hadCutRanges) { 374 al.clear(); 375 } 376 377 logger.debug("Total scans cut this granule: " + cutScanCount); 378 379 } 380 } 381 } 382 granCutScans.put(granuleIndex, new Integer(cutScanCount)); 383 granCutRanges.put(granuleIndex, al); 384 } 385 386 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 387 388 ncfile = nclist.get(ncIdx); 389 390 Map<String, Variable> varMap = new HashMap<>(); 391 Map<String, String[]> varDimNames = new HashMap<>(); 392 Map<String, DataType> varDataType = new HashMap<>(); 393 394 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 395 int varInTrackIndex = -1; 396 while (varIter.hasNext()) { 397 Variable var = varIter.next(); 398 399 boolean foundProduct = false; 400 for (String s : products) { 401 if (s.contains(var.getFullName())) { 402 logger.trace("Valid product: " + var.getFullName()); 403 foundProduct = true; 404 break; 405 } 406 } 407 408 if (! foundProduct) { 409 logger.trace("Skipping variable: " + var.getFullName()); 410 continue; 411 } 412 413 if (var instanceof Structure) { 414 // simply skip these, applicable only to IASI far as I know 415 continue; 416 } 417 418 int rank = var.getRank(); 419 420 // bypass any less-than-2D variables for now... 421 if (rank < 2) { 422 continue; 423 } 424 425 String varName = var.getFullName(); 426 varMap.put(varName, var); 427 Iterator<Dimension> dimIter = var.getDimensions().iterator(); 428 String[] dimNames = new String[rank]; 429 int[] dimLengths = new int[rank]; 430 int cnt = 0; 431 boolean notDisplayable = false; 432 varInTrackIndex = getInTrackIndex(var); 433 434 while (dimIter.hasNext()) { 435 Dimension dim = dimIter.next(); 436 String s = dim.getShortName(); 437 if ((s != null) && (!s.isEmpty())) { 438 if ((! s.equals(inTrackDimensionName)) && 439 ((! s.startsWith("Band")) && (cnt == 0)) && 440 (! varName.endsWith("Latitude")) && 441 (! varName.endsWith("latitude")) && 442 (! varName.endsWith("Latitude_TC")) && 443 (! varName.endsWith("Longitude")) && 444 (! varName.endsWith("longitude")) && 445 (! varName.endsWith("Longitude_TC")) && 446 (! s.equals(crossTrackDimensionName))) { 447 notDisplayable = true; 448 break; 449 } 450 } 451 String dimName = dim.getShortName(); 452 logger.debug("GranuleAggregation init, variable: " + varName + ", dimension name: " + dimName + ", length: " + dim.getLength()); 453 if (dimName == null) dimName = "dim" + cnt; 454 if (dimName.isEmpty()) { 455 dimName = "dim" + cnt; 456 } 457 dimNames[cnt] = dimName; 458 dimLengths[cnt] = dim.getLength(); 459 cnt++; 460 } 461 462 // skip to next variable if it's not displayable data 463 if (notDisplayable) continue; 464 465 // adjust in-track dimension if needed (scans were cut) 466 int cutScans = 0; 467 if (! granCutScans.isEmpty() && granCutScans.containsKey(ncIdx)) { 468 cutScans = granCutScans.get(new Integer(ncIdx)); 469 } else { 470 granCutScans.put(new Integer(ncIdx), new Integer(0)); 471 } 472 473 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] - cutScans; 474 475 // XXX TJJ - can below block go away? Think so... 476 int[] aggrDimLengths = varAggrDimLengths.get(varName); 477 for (int i = 0; i < rank; i++) { 478 if (i == varInTrackIndex) { 479 aggrDimLengths[i] += dimLengths[i]; 480 } else { 481 aggrDimLengths[i] = dimLengths[i]; 482 } 483 } 484 485 varDimNames.put(varName, dimNames); 486 varDataType.put(varName, var.getDataType()); 487 488 if (varInTrackIndex < 0) { 489 logger.debug("Skipping variable with unknown dimension: " + var.getFullName()); 490 continue; 491 } 492 493 Map<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(varName); 494 granIdxToInTrackLen.put(ncIdx, new Integer(dimLengths[varInTrackIndex])); 495 496 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] * granuleCount; 497 varDataType.put(varName, var.getDataType()); 498 } 499 500 // add the new hashmaps to our enclosing lists 501 varMapList.add(varMap); 502 varDimNamesList.add(varDimNames); 503 varDataTypeList.add(varDataType); 504 505 } 506 } 507 508 /** 509 * Based on the names of the variable dimensions, determine the in-track index. 510 * 511 * @param v {@code Variable} that {@literal "contains"} dimension names that 512 * allow for inference of the in-track index. {@code null} is allowed. 513 * 514 * @return correct index (0 or greater), or -1 if error. 515 */ 516 517 private int getInTrackIndex(Variable v) { 518 519 int index = -1; 520 boolean is2D = false; 521 boolean is3D = false; 522 523 String inTrackName = null; 524 525 // typical sanity check 526 if (v == null) return index; 527 528 // lat/lon vars have different dimension names 529 // Set a default that will work for Enterprise EDRs 530 inTrackName = inTrackGeoDimensionName; 531 if ((v.getFullName().endsWith("Latitude")) || 532 (v.getFullName().endsWith("Latitude_TC")) || 533 (v.getFullName().endsWith("Longitude")) || 534 (v.getFullName().endsWith("latitude")) || 535 (v.getFullName().endsWith("longitude")) || 536 (v.getFullName().endsWith("Longitude_TC"))) { 537 if ((v.getFullName().startsWith("All_Data")) || 538 (v.getFullName().startsWith("observation_data"))) { 539 inTrackName = inTrackDimensionName; 540 } else { 541 inTrackName = inTrackGeoDimensionName; 542 } 543 } else { 544 inTrackName = inTrackDimensionName; 545 } 546 547 // pull out the dimensions 548 List<Dimension> dList = v.getDimensions(); 549 550 // right now, we only handle 2D and 3D variables. 551 // TJJ XXX it does get trickier, and we will have to expand this 552 // to deal with for example CrIS data... 553 int numDimensions = dList.size(); 554 555 // the only 4D data right now is CrIS, return 0 556 if (numDimensions == 4) return 0; 557 558 if ((numDimensions == 2) || (numDimensions == 3)) { 559 if (numDimensions == 2) is2D = true; 560 if (numDimensions == 3) is3D = true; 561 } else { 562 return index; 563 } 564 565 // if the data is 2D, we use the SwathAdapter class, 566 // if 3D, we use the SpectrumAdapter class 567 for (int i = 0; i < numDimensions; i++) { 568 if (is2D) { 569 // XXX TJJ - if empty name, in-track index is 0 570 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 571 logger.trace("Empty dimension name!, assuming in-track dim is 0"); 572 return 0; 573 } 574 if (dList.get(i).getShortName().equals(inTrackName)) { 575 index = i; 576 break; 577 } 578 } 579 if (is3D) { 580 // XXX TJJ - if empty name, in-track index is 0 581 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 582 logger.debug("Empty dimension name!, assuming in-track dim is 0"); 583 return 0; 584 } 585 if (dList.get(i).getShortName().equals(inTrackName)) { 586 index = i; 587 break; 588 } 589 } 590 } 591 592 // hopefully we found the right one 593 return index; 594 } 595 596 private synchronized Object readArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 597 598 String mapName = array_name; 599 array_name = mapNameIfQualityFlag(array_name); 600 array_name = mapNameIfLUTVar(array_name); 601 // how many dimensions are we dealing with 602 int dimensionCount = start.length; 603 604 // pull out a representative variable so we can determine which index is in-track 605 Variable vTmp = varMapList.get(0).get(array_name); 606 logger.trace(""); 607 logger.trace("Working on var: " + array_name); 608 int vInTrackIndex = getInTrackIndex(vTmp); 609 610 int loGranuleId = 0; 611 int hiGranuleId = 0; 612 613 Map<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(array_name); 614 int numGrans = granIdxToInTrackLen.size(); 615 616 int[] vGranuleLengths = new int[numGrans]; 617 for (int k = 0; k < numGrans; k++) { 618 vGranuleLengths[k] = granIdxToInTrackLen.get(k); 619 logger.debug("readArray, gran len: " + vGranuleLengths[k] + ", scans cut: " + granCutScans.get(k)); 620 } 621 622 int strt = start[vInTrackIndex]; 623 int stp = strt + (count[vInTrackIndex] - 1) * stride[vInTrackIndex]; 624 int cnt = 0; 625 for (int k = 0; k < numGrans; k++) { 626 int granLen = granIdxToInTrackLen.get(k); 627 cnt += granLen; 628 if (strt < cnt) { 629 loGranuleId = k; 630 break; 631 } 632 } 633 634 cnt = 0; 635 for (int k = 0; k < numGrans; k++) { 636 int granLen = granIdxToInTrackLen.get(k); 637 cnt += granLen; 638 if (stp < cnt) { 639 hiGranuleId = k; 640 break; 641 } 642 } 643 644 int totalScansCut = 0; 645 for (int k = loGranuleId; k <= hiGranuleId; k++) { 646 totalScansCut += granCutScans.get(k); 647 } 648 logger.debug("Scans cut for this selection: " + totalScansCut); 649 650 // next, we break out the offsets, counts, and strides for each granule 651 int granuleSpan = hiGranuleId - loGranuleId + 1; 652 653 logger.debug("readArray req, loGran: " + loGranuleId + ", hiGran: " + 654 hiGranuleId + ", granule span: " + granuleSpan + ", dimCount: " + dimensionCount); 655 656 for (int i = 0; i < dimensionCount; i++) { 657 logger.debug("start[" + i + "]: " + start[i]); 658 logger.debug("count[" + i + "]: " + count[i]); 659 logger.debug("stride[" + i + "]: " + stride[i]); 660 } 661 662 int [][] startSet = new int [granuleSpan][dimensionCount]; 663 int [][] countSet = new int [granuleSpan][dimensionCount]; 664 int [][] strideSet = new int [granuleSpan][dimensionCount]; 665 int countSubtotal = 0; 666 667 int inTrackTotal = 0; 668 for (int i = 0; i < loGranuleId; i++) { 669 inTrackTotal += vGranuleLengths[i]; 670 } 671 672 // this part is a little tricky - set the values for each granule we need to access for this read 673 for (int i = 0; i < granuleSpan; i++) { 674 675 inTrackTotal += vGranuleLengths[loGranuleId + i]; 676 677 for (int j = 0; j < dimensionCount; j++) { 678 // for all indeces other than the in-track index, the numbers match what was passed in 679 if (j != vInTrackIndex) { 680 startSet[i][j] = start[j]; 681 countSet[i][j] = count[j] * stride[j]; 682 strideSet[i][j] = stride[j]; 683 } else { 684 // for the in-track index, it's not so easy... 685 // for first granule, start is what's passed in 686 if (i == 0) { 687 if (inTrackTotal > vGranuleLengths[loGranuleId + i]) { 688 startSet[i][j] = 689 (start[j] % (inTrackTotal - vGranuleLengths[loGranuleId + i])); 690 } else { 691 startSet[i][j] = start[j]; 692 } 693 } else { 694 startSet[i][j] = 695 ((inTrackTotal - (vGranuleLengths[loGranuleId + i])) % stride[j]); 696 // if there is a remainder, need to offset stride - remainder into next gran 697 if (startSet[i][j] != 0) startSet[i][j] = stride[j] - startSet[i][j]; 698 } 699 700 // counts may be different for start, end, and middle granules 701 if (i == 0) { 702 // is this the first and only granule? 703 if (granuleSpan == 1) { 704 countSet[i][j] = count[j] * stride[j]; 705 // TJJ May 2016 706 // This condition manifests because there are times when 707 // "fill" scans are cut from otherwise fine granules. 708 // e.g., to the chooser it may look like there are 3072 valid lines, 709 // but by the time we get here we realize there are really 3056 710 // This typically shortens the granule by one scan (16 lines) 711 if (countSet[i][j] > (vGranuleLengths[loGranuleId+i])) 712 countSet[i][j] = vGranuleLengths[loGranuleId+i]; 713 714 // or is this the first of multiple granules... 715 } else { 716 if ((inTrackTotal - start[j]) < (count[j] * stride[j])) { 717 countSet[i][j] = inTrackTotal - start[j]; 718 } else { 719 countSet[i][j] = count[j] * stride[j]; 720 } 721 countSubtotal += countSet[i][j]; 722 } 723 } else { 724 // middle granules 725 if (i < (granuleSpan - 1)) { 726 countSet[i][j] = vGranuleLengths[loGranuleId+i] - startSet[i][j]; 727 countSubtotal += countSet[i][j]; 728 } else { 729 // the end granule 730 countSet[i][j] = (count[j] * stride[j]) - countSubtotal - startSet[i][j]; 731 // TJJ May 2016 732 // This condition manifests because there are times when 733 // "fill" scans are cut from otherwise fine granules. 734 // e.g., to the chooser it may look like there are 3072 valid lines, 735 // but by the time we get here we realize there are really 3056 736 // This typically shortens the granule by one scan (16 lines) 737 if (countSet[i][j] > (vGranuleLengths[loGranuleId+i] - startSet[i][j])) 738 countSet[i][j] = vGranuleLengths[loGranuleId+i] - startSet[i][j]; 739 } 740 } 741 // luckily, stride never changes 742 strideSet[i][j] = stride[j]; 743 } 744 } 745 } 746 747 int totalLength = 0; 748 int rangeListCount = 0; 749 List<Array> arrayList = new ArrayList<>(); 750 for (int granuleIdx = 0; granuleIdx < granuleCount; granuleIdx++) { 751 if ((granuleIdx >= loGranuleId) && (granuleIdx <= hiGranuleId)) { 752 Variable var = varMapList.get(loGranuleId + (granuleIdx-loGranuleId)).get(array_name); 753 754 if (var instanceof Structure) { 755 // what to do here? 756 } else { 757 List<Range> rangeList = new ArrayList<>(); 758 for (int dimensionIdx = 0; dimensionIdx < dimensionCount; dimensionIdx++) { 759 logger.debug("Creating new Range: " + startSet[rangeListCount][dimensionIdx] + 760 ", " + (startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1) + ", " + strideSet[rangeListCount][dimensionIdx]); 761 Range range = new Range( 762 startSet[rangeListCount][dimensionIdx], 763 startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1, 764 strideSet[rangeListCount][dimensionIdx] 765 ); 766 rangeList.add(dimensionIdx, range); 767 } 768 rangeListCount++; 769 770 // If there were chunks of fill data to remove... 771 List<Range> al = granCutRanges.get(new Integer(granuleIdx)); 772 if (! al.isEmpty()) { 773 List<Variable> varChunks = new ArrayList<>(); 774 for (int rangeCount = 0; rangeCount < al.size(); rangeCount+=2) { 775 List<Range> rl = new ArrayList<>(); 776 rl.add(al.get(rangeCount)); 777 rl.add(al.get(rangeCount + 1)); 778 varChunks.add(var.section(rl)); 779 } 780 781 int [] newShape = var.getShape(); 782 int cutScans = granCutScans.get(granuleIdx); 783 newShape[0] = newShape[0] - cutScans; 784 logger.trace("New Shape: " + newShape[0] + ", " + newShape[1]); 785 Array single = Array.factory(var.getDataType(), newShape); 786 787 // now read variable chunk data into single contiguous array 788 int idx = 0; 789 for (Variable v : varChunks) { 790 Array data = v.read(); 791 int [] tmpShape = v.getShape(); 792 for (int tIdx = 0; tIdx < tmpShape.length; tIdx++) { 793 logger.trace("Shape[" + tIdx + "]: " + tmpShape[tIdx]); 794 } 795 IndexIterator ii = data.getIndexIterator(); 796 while (ii.hasNext()) { 797 single.setFloat(idx, ii.getFloatNext()); 798 idx++; 799 } 800 } 801 802 // finally, apply subset ranges 803 logger.debug("Size of cut src array: " + single.getSize()); 804 Array subarray = single.section(rangeList); 805 totalLength += subarray.getSize(); 806 arrayList.add(subarray); 807 logger.debug("Size of cut sub array: " + subarray.getSize()); 808 809 } else { 810 Array subarray = var.read(rangeList); 811 totalLength += subarray.getSize(); 812 logger.debug("Size of reg sub array: " + subarray.getSize()); 813 arrayList.add(subarray); 814 } 815 816 } 817 // put in an empty ArrayList placeholder to retain a slot for each granule 818 } else { 819 Array emptyArray = null; 820 arrayList.add(emptyArray); 821 } 822 } 823 824 // last, concatenate the individual NetCDF arrays pulled out 825 826 DataType arrayType = getArrayType(array_name); 827 RangeProcessor rngProcessor = varToRangeProcessor.get(array_name); 828 829 logger.debug("Creating aggregated array, totalLength: " + totalLength); 830 831 // TJJ May 2016 832 // I'm starting to think there may be a bug in the Java NetCDF section/subarray 833 // code - I have chased this quite a bit with no solution, the total length 834 // sectioned out sometimes exceeds the total requested. It's almost as if a 835 // previous section length is retained. Anyway, as a hack for now I will just 836 // truncate if this occurs. We also need to watch for overflow in the arraycopy 837 // calls below 838 839 if (count.length < 3) { 840 if (totalLength > (count[0] * count[1])) { 841 totalLength = count[0] * count[1]; 842 } 843 } 844 845 float[] finalArray = new float[totalLength]; 846 847 int destPos = 0; 848 int granIdx = 0; 849 850 int remaining = totalLength; 851 for (Array a : arrayList) { 852 if (a != null) { 853 Object primArray = a.copyTo1DJavaArray(); 854 primArray = processArray( 855 mapName, array_name, arrayType, granIdx, primArray, rngProcessor, start, count 856 ); 857 if (a.getSize() > remaining) { 858 System.arraycopy(primArray, 0, finalArray, destPos, remaining); 859 } else { 860 System.arraycopy(primArray, 0, finalArray, destPos, (int) a.getSize()); 861 } 862 destPos += a.getSize(); 863 remaining -= (int) a.getSize(); 864 } 865 granIdx++; 866 } 867 868 return finalArray; 869 } 870 871 /** 872 * @param qfMap the qfMap to set 873 */ 874 875 public void setQfMap(Map<String, QualityFlag> qfMap) { 876 this.qfMap = qfMap; 877 } 878 879 /** 880 * @param lutMap the lutMap to set 881 */ 882 883 public void setLUTMap(Map<String, float[]> lutMap) { 884 this.lutMap = lutMap; 885 } 886 887 public Map<String, Variable> getVarMap() { 888 return varMapList.get(0); 889 } 890 891 public List<NetCDFFile> getReaders() { 892 return this.ncdfal; 893 } 894 895 /* pass individual granule pieces just read from dataset through the RangeProcessor */ 896 private Object processArray(String mapName, String array_name, DataType arrayType, int granIdx, Object values, RangeProcessor rngProcessor, int[] start, int[] count) { 897 898 if (rngProcessor == null) { 899 return values; 900 } 901 else { 902 ((AggregationRangeProcessor) rngProcessor).setWhichRangeProcessor(granIdx); 903 904 Object outArray = null; 905 906 if (arrayType == DataType.SHORT) { 907 // if variable is a LUT var, apply LUT 908 if ((lutMap != null) && (lutMap.containsKey(mapName))) { 909 float lut[] = lutMap.get(mapName); 910 outArray = rngProcessor.processRangeApplyLUT((short []) values, lut); 911 } else { 912 outArray = rngProcessor.processRange((short[]) values, null); 913 } 914 } else if (arrayType == DataType.BYTE) { 915 // if variable is a bit-field quality flag, apply mask 916 if ((qfMap != null) && (qfMap.containsKey(origName))) { 917 QualityFlag qf = qfMap.get(origName); 918 outArray = rngProcessor.processRangeQualityFlag((byte[]) values, null, qf); 919 } else { 920 outArray = rngProcessor.processRange((byte[]) values, null); 921 } 922 } else if (arrayType == DataType.FLOAT) { 923 outArray = rngProcessor.processRange((float[]) values, null); 924 } else if (arrayType == DataType.DOUBLE) { 925 outArray = rngProcessor.processRange((double[]) values, null); 926 } 927 928 return outArray; 929 } 930 } 931 932 /* Application can supply a RangeProcessor for a variable 'arrayName' */ 933 public void addRangeProcessor(String arrayName, RangeProcessor rangeProcessor) { 934 varToRangeProcessor.put(arrayName, rangeProcessor); 935 } 936 937}