/*
* Copyright 1998-2014 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.grib;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import ucar.ma2.Array;
import ucar.ma2.ArrayDouble;
import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.dataset.*;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.grib.collection.GribCdmIndex;
import ucar.nc2.grib.collection.GribCollectionImmutable;
import ucar.nc2.grib.collection.GribIosp;
import ucar.nc2.grib.collection.PartitionCollectionImmutable;
import ucar.nc2.grib.grib2.Grib2Pds;
import ucar.nc2.grib.grib2.Grib2Record;
import ucar.nc2.grib.grib2.table.Grib2Customizer;
import ucar.nc2.iosp.IOServiceProvider;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.util.DebugFlagsImpl;
import ucar.nc2.util.Misc;
import ucar.nc2.util.cache.FileCache;
import ucar.nc2.util.cache.FileCacheIF;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.util.test.category.NeedsCdmUnitTest;
import ucar.unidata.util.test.TestDir;
import java.io.IOException;
import java.util.Formatter;
/**
* Describe
*
* @author caron
* @since 11/7/2014
*/
@Category(NeedsCdmUnitTest.class)
public class TestGrib2CoordsMatch {
@BeforeClass
static public void before() {
GribIosp.debugIndexOnlyCount = 0;
GribCollectionImmutable.countGC = 0;
PartitionCollectionImmutable.countPC = 0;
RandomAccessFile.enableDefaultGlobalFileCache();
RandomAccessFile.setDebugLeaks(true);
GribCdmIndex.setGribCollectionCache(new ucar.nc2.util.cache.FileCacheGuava("GribCollectionCacheGuava", 100));
GribCdmIndex.gribCollectionCache.resetTracking();
}
@AfterClass
static public void after() {
GribIosp.setDebugFlags(new DebugFlagsImpl());
Formatter out = new Formatter(System.out);
FileCacheIF cache = GribCdmIndex.gribCollectionCache;
if (cache != null) {
cache.showTracking(out);
cache.showCache(out);
cache.clearCache(false);
}
FileCacheIF rafCache = RandomAccessFile.getGlobalFileCache();
if (rafCache != null) {
rafCache.showCache(out);
}
System.out.printf(" countGC=%7d%n", GribCollectionImmutable.countGC);
System.out.printf(" countPC=%7d%n", PartitionCollectionImmutable.countPC);
System.out.printf(" countDataAccess=%7d%n", GribIosp.debugIndexOnlyCount);
System.out.printf(" total files needed=%7d%n", GribCollectionImmutable.countGC + PartitionCollectionImmutable.countPC + GribIosp.debugIndexOnlyCount);
FileCache.shutdown();
RandomAccessFile.setGlobalFileCache(null);
TestDir.checkLeaks();
RandomAccessFile.setDebugLeaks(false);
}
@BeforeClass
public static void setup() {
Grib2Record.getlastRecordRead = true;
}
@AfterClass
public static void cleanup() {
Grib2Record.getlastRecordRead = false;
}
@Test
public void problem() throws IOException {
long start = System.currentTimeMillis();
// GribIosp.setDebugFlags(new DebugFlagsImpl("Grib/indexOnly Grib/indexOnlyShow"));
String filename = "gribCollections/gfs_2p5deg/GFS_Global_2p5deg_20150301_0000.grib2.ncx3";
try (GridDataset gds = GridDataset.open(TestDir.cdmUnitTestDir + filename)) {
NetcdfFile ncfile = gds.getNetcdfFile();
IOServiceProvider iosp = ncfile.getIosp();
assert iosp instanceof GribIosp;
iospGrib = (GribIosp) iosp;
GridDatatype gdt = gds.findGridByName("Geopotential_height_potential_vorticity_surface");
assert gdt != null;
TestGribCollections.Count count = read(gdt);
System.out.printf("%n%50s == %d/%d%n", "total", count.nmiss, count.nread);
long took = System.currentTimeMillis() - start;
float r = ((float) took) / count.nread;
System.out.printf("%n that took %d secs total, %f msecs per record%n", took / 1000, r);
assert count.nread == 186;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
}
@Test
public void testDgexSRC() throws IOException {
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/dgex/20141011/DGEX_CONUS_12km_20141011_0600.grib2.ncx3");
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
assert count.nread == 1009;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
@Test
public void testDgexTP() throws IOException {
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/dgex/20141011/dgex_46-20141011.ncx3");
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
assert count.nread == 3140;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
@Test
public void testDgexPofP() throws IOException {
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/dgex/dgex_46.ncx3");
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
// 2015/03/11: These tests were commented out, causing this test to be a no-op. Why?
assert count.nread == 5384;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
@Test
public void testCfsrSingleFile() throws IOException {
// CFSR dataset: 0-6 hour forecasts x 124 runtimes (4x31)
// there are 2 groups, likely miscoded, the smaller group has duplicate 0 hour, probably miscoded
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/cfsr/cfrsAnalysis_46.ncx3");
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
assert count.nread == 868;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
@Test
public void testGC() throws IOException {
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/GFS_Global_2p5deg_20150301_0000.grib2.ncx3");
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
//assert count.nread == 22909; // 0/2535/23229 or 150/2535/22909
// assert count.nmiss == 2535;
//assert count.nerrs == 150;
assert count.nread == 33994;
assert count.nmiss == 0;
assert count.nerrs == 0;
}
@Test
@Ignore("test takes 45 minutes on jenkins - turn off for now")
public void testPofG() throws IOException { //ncss/GFS/CONUS_80km/GFS_CONUS_80km-CONUS_80km.ncx2
TestGribCollections.Count count = read(TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx3");
// that took 2497 secs total, 26.835802 msecs per record total == 671/10296/93052
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
assert count.nread == 93052;
assert count.nmiss == 10296;
assert count.nerrs == 671;
}
@Test
@Ignore("only works when we have the data - no \"indexOnly\" mode")
public void openFileProblem() throws IOException {
long start = System.currentTimeMillis();
String filename = TestDir.cdmUnitTestDir + "gribCollections/rdavm/ds083.2/grib1/ds083.2_Aggregation-grib1.ncx3";
try (GridDataset gds = GridDataset.open(filename)) {
GridDatatype gdt = gds.findGridByName("Best/Land_cover_land1_sea0_surface");
assert gdt != null;
gdc = gdt.getCoordinateSystem();
NetcdfFile ncfile = gds.getNetcdfFile();
IOServiceProvider iosp = ncfile.getIosp();
assert iosp instanceof GribIosp;
iospGrib = (GribIosp) iosp;
TestGribCollections.Count count = new TestGribCollections.Count();
int n = 1000;
int first = 5500;
readTimeRange(gdt, count, first, n);
long took = System.currentTimeMillis() - start;
float r = ((float) took) / n;
System.out.printf("%n that took %d secs total, %d records %f msecs per record%n", took / 1000, n, r);
System.out.printf("%n%50s == %d/%d/%d%n", "total", count.nerrs, count.nmiss, count.nread);
//assert count.nread == 81340;
//assert count.nmiss == 1801;
//assert count.nerrs == 0;
}
}
///////////////////////////////////////////////////////////////
private GribIosp iospGrib;
private TestGribCollections.Count read(String filename) {
long start = System.currentTimeMillis();
System.out.println("\n\nReading File " + filename);
TestGribCollections.Count allCount = new TestGribCollections.Count();
try (GridDataset gds = GridDataset.open(filename)) {
NetcdfFile ncfile = gds.getNetcdfFile();
IOServiceProvider iosp = ncfile.getIosp();
assert iosp instanceof GribIosp;
iospGrib = (GribIosp) iosp;
for (GridDatatype gdt: gds.getGrids()) {
TestGribCollections.Count count = read(gdt);
System.out.printf("%80s == %d/%d%n", gdt.getFullName(), count.nmiss, count.nread);
allCount.add(count);
}
long took = System.currentTimeMillis() - start;
float r = ((float) took) / allCount.nread;
System.out.printf("%n%80s == %d/%d%n", "total", allCount.nmiss, allCount.nread);
System.out.printf("%n that took %d secs total, %f msecs per record%n", took/1000, r);
} catch (IOException ioe) {
System.out.printf("%s%n", ioe);
Formatter out = new Formatter(System.out);
GribCdmIndex.gribCollectionCache.showCache(out);
}
return allCount;
}
private GridCoordSystem gdc;
private String var_desc;
private String var_param;
private String var_level_type;
private CalendarDate runtimeCoord;
private ArrayDouble.D3 timeCoord2DBoundsArray;
private TestGribCollections.Count read(GridDatatype gdt) throws IOException {
var_desc = gdt.findAttValueIgnoreCase("Grib2_Parameter_Name", "");
Attribute paramAtt = gdt.findAttributeIgnoreCase("Grib2_Parameter");
int disc = paramAtt.getNumericValue(0).intValue();
int cat = paramAtt.getNumericValue(1).intValue();
int num = paramAtt.getNumericValue(2).intValue();
var_param = disc + "-" + cat + "-" + num;
var_level_type = gdt.findAttValueIgnoreCase("Grib2_Level_Type", "");
gdc = gdt.getCoordinateSystem();
Dimension rtDim = gdt.getRunTimeDimension();
Dimension tDim = gdt.getTimeDimension();
Dimension zDim = gdt.getZDimension();
// loop over runtime
TestGribCollections.Count count = new TestGribCollections.Count();
if (rtDim != null) {
CoordinateAxis1DTime rtcoord = gdc.getRunTimeAxis();
for (int rt=0; rt<rtDim.getLength(); rt++) {
runtimeCoord = rtcoord.getCalendarDate(rt);
readTime(gdt, count, rt, tDim, zDim);
}
} else {
runtimeCoord = null;
readTime(gdt, count, -1, tDim, zDim);
}
timeCoord2DBoundsArray = null;
return count;
}
private boolean hasTime;
private double timeCoord;
private double[] timeBounds;
private boolean isTimeInterval;
private CalendarDate timeCoordDate;
private CalendarDate[] timeBoundsDate; // for intervals
private void readTimeRange(GridDatatype gdt, TestGribCollections.Count count, int start, int n) throws IOException {
hasTime = true;
CoordinateAxis tcoord = gdc.getTimeAxis();
isTimeInterval = tcoord.isInterval();
assert (tcoord instanceof CoordinateAxis1DTime);
CoordinateAxis1DTime tcoord1D = (CoordinateAxis1DTime) tcoord;
for (int t = start; t < start + n; t++) {
if (isTimeInterval) {
timeBounds = tcoord1D.getCoordBounds(t);
timeBoundsDate = tcoord1D.getCoordBoundsDate(t);
} else {
timeCoord = tcoord1D.getCoordValue(t);
timeCoordDate = tcoord1D.getCalendarDate(t);
}
readVert(gdt, count, -1, t, null);
}
}
private void readTime(GridDatatype gdt, TestGribCollections.Count count, int rtIndex, Dimension timeDim, Dimension zDim) throws IOException {
if (timeDim != null) {
hasTime = true;
CoordinateAxis tcoord = gdc.getTimeAxis();
isTimeInterval = tcoord.isInterval();
if (rtIndex < 0) {
assert (tcoord instanceof CoordinateAxis1DTime);
CoordinateAxis1DTime tcoord1D = (CoordinateAxis1DTime) tcoord;
for (int t = 0; t < timeDim.getLength(); t++) {
if (isTimeInterval) {
timeBounds = tcoord1D.getCoordBounds(t);
timeBoundsDate = tcoord1D.getCoordBoundsDate(t);
} else {
timeCoord = tcoord1D.getCoordValue(t);
timeCoordDate = tcoord1D.getCalendarDate(t);
}
readVert(gdt, count, rtIndex, t, zDim);
}
} else {
assert (tcoord instanceof CoordinateAxis2D);
CoordinateAxis2D tcoord2D = (CoordinateAxis2D) tcoord;
CoordinateAxisTimeHelper helper = tcoord2D.getCoordinateAxisTimeHelper();
if (timeCoord2DBoundsArray == null)
timeCoord2DBoundsArray = tcoord2D.getCoordBoundsArray();
for (int t = 0; t < timeDim.getLength(); t++) {
if (isTimeInterval) {
timeBounds = new double[2];
timeBounds[0] = timeCoord2DBoundsArray.get(rtIndex, t, 0);
timeBounds[1] = timeCoord2DBoundsArray.get(rtIndex, t, 1);
timeBoundsDate = new CalendarDate[2];
timeBoundsDate[0] = helper.makeCalendarDateFromOffset(timeBounds[0]);
timeBoundsDate[1] = helper.makeCalendarDateFromOffset(timeBounds[1]);
} else {
timeCoord = tcoord2D.getCoordValue(rtIndex, t);
timeCoordDate = helper.makeCalendarDateFromOffset(timeCoord);
}
readVert(gdt, count, rtIndex, t, zDim);
}
}
} else {
hasTime = false;
readVert(gdt, count, rtIndex, -1, zDim);
}
}
private boolean hasVert;
private double vertCoord;
private double[] edge;
private boolean isLayer;
private void readVert(GridDatatype gdt, TestGribCollections.Count count, int rtIndex, int tIndex, Dimension zDim) throws IOException {
if (zDim != null) {
hasVert = true;
CoordinateAxis1D zcoord = gdc.getVerticalAxis();
isLayer = zcoord.isInterval();
for (int z=0; z<zDim.getLength(); z++) {
if (isLayer) {
edge = zcoord.getCoordBounds(z);
} else {
vertCoord = zcoord.getCoordValue(z);
}
readAndTest(gdt, count, rtIndex, tIndex, z);
}
} else {
hasVert = false;
readAndTest(gdt, count, rtIndex, tIndex, -1);
}
}
private boolean show = false;
private void readAndTest(GridDatatype gdt, TestGribCollections.Count count, int rtIndex, int tIndex, int zIndex) throws IOException {
iospGrib.clearLastRecordRead();
Array data = gdt.readDataSlice(rtIndex, -1, tIndex, zIndex, -1, -1);
Grib2Customizer cust = (Grib2Customizer) iospGrib.getGribCustomizer();
Grib2Record grib2 = (Grib2Record) iospGrib.getLastRecordRead();
if (grib2 == null) {
count.nmiss++;
count.nread++;
return;
}
Grib2RecordBean bean = new Grib2RecordBean(cust, grib2);
boolean paramOk = true;
paramOk &= var_desc.equals(bean.getName());
paramOk &= var_param.equals(bean.getParamNo());
paramOk &= var_level_type.equals(bean.getLevelName());
boolean runtimeOk = true;
CalendarDate gribDate;
if (runtimeCoord != null) {
gribDate = bean.getRefDate();
runtimeOk &= runtimeCoord.equals(gribDate);
}
boolean timeOk = true;
if (hasTime) {
if (isTimeInterval) {
timeOk &= bean.isTimeInterval();
TimeCoord.TinvDate dateFromGribRecord = bean.getTimeIntervalDates();
timeOk &= timeBoundsDate[0].equals(dateFromGribRecord.getStart());
timeOk &= timeBoundsDate[1].equals(dateFromGribRecord.getEnd());
} else {
// timeOk &= timeCoord == bean.getTimeCoordValue(); // true if GC
CalendarDate dateFromGribRecord = bean.getForecastDate();
timeOk &= timeCoordDate.equals(dateFromGribRecord);
}
}
boolean vertOk = true;
if (hasVert) {
if (isLayer) {
vertOk &= bean.isLayer();
vertOk &= Misc.closeEnough(edge[0],bean.getLevelLowValue());
vertOk &= Misc.closeEnough(edge[1],bean.getLevelHighValue());
} else {
vertOk &= Misc.closeEnough(vertCoord, bean.getLevelValue1());
}
}
boolean ok = paramOk && runtimeOk && timeOk && vertOk;
if (show || !ok) {
System.out.printf("%s%n", bean);
}
if (!ok)
count.nerrs++;
count.nread++;
}
public class Grib2RecordBean {
Grib2Customizer cust;
Grib2Record gr;
Grib2Pds pds;
int discipline;
public Grib2RecordBean() {
}
public Grib2RecordBean(Grib2Customizer cust, Grib2Record gr) throws IOException {
this.cust = cust;
this.gr = gr;
this.pds = gr.getPDS();
discipline = gr.getDiscipline();
}
public String getParamNo() {
return discipline + "-" + pds.getParameterCategory() + "-" + pds.getParameterNumber();
}
public String getName() {
return cust.getVariableName(gr);
}
public String getLevelName() {
return cust.getLevelName(pds.getLevelType1());
}
public final CalendarDate getRefDate() {
return gr.getReferenceDate();
}
public boolean isLayer() {
return cust.isLayer(pds);
}
public final CalendarDate getForecastDate() {
return cust.getForecastDate(gr);
}
public String getTimeCoord() {
if (isTimeInterval())
return getTimeIntervalDates().toString();
else
return getForecastDate().toString();
}
public final String getTimeUnit() {
int unit = pds.getTimeUnit();
return cust.getTableValue("4.4", unit);
}
public final int getForecastTime() {
return pds.getForecastTime();
}
public String getLevel() {
int v1 = pds.getLevelType1();
int v2 = pds.getLevelType2();
if (v1 == 255) return "";
if (v2 == 255) return "" + pds.getLevelValue1();
if (v1 != v2) return pds.getLevelValue1() + "-" + pds.getLevelValue2() + " level2 type= " + v2;
return pds.getLevelValue1() + "-" + pds.getLevelValue2();
}
public double getLevelValue1() {
return pds.getLevelValue1();
}
public double getLevelLowValue() {
return Math.min(pds.getLevelValue1(), pds.getLevelValue2());
}
public double getLevelHighValue() {
return Math.max(pds.getLevelValue1(), pds.getLevelValue2());
}
/////////////////////////////////////////////////////////////
/// time intervals
public boolean isTimeInterval() {
return pds instanceof Grib2Pds.PdsInterval;
}
public TimeCoord.TinvDate getTimeIntervalDates() {
if (cust != null && isTimeInterval()) {
return cust.getForecastTimeInterval(gr);
}
return null;
}
@Override
public String toString() {
final Formatter sb = new Formatter();
sb.format("Record dataStart=%s%n", gr.getDataSection().getStartingPosition());
sb.format(" %s (%s)%n", getName(), getParamNo());
sb.format(" reftime=%s%n", getRefDate());
sb.format(" time=%s%n", getTimeCoord());
sb.format(" level=%s type=%s (%d)%n", getLevel(), getLevelName(), pds.getLevelType1());
return sb.toString();
}
///////////////////////////////
// Ensembles
public int getPertN() {
Grib2Pds.PdsEnsemble pdsi = (Grib2Pds.PdsEnsemble) pds;
int v = pdsi.getPerturbationNumber();
if (v == GribNumbers.UNDEFINED) v = -1;
return v;
}
public int getNForecastsInEns() {
Grib2Pds.PdsEnsemble pdsi = (Grib2Pds.PdsEnsemble) pds;
int v = pdsi.getNumberEnsembleForecasts();
if (v == GribNumbers.UNDEFINED) v = -1;
return v;
}
public int getPertType() {
Grib2Pds.PdsEnsemble pdsi = (Grib2Pds.PdsEnsemble) pds;
int v = pdsi.getPerturbationType();
return (v == GribNumbers.UNDEFINED) ? -1 : v;
}
/////////////////////////////////
// Probability
public String getProbLimits() {
Grib2Pds.PdsProbability pdsi = (Grib2Pds.PdsProbability) pds;
double v = pdsi.getProbabilityLowerLimit();
if (v == GribNumbers.UNDEFINEDD) return "";
else return pdsi.getProbabilityLowerLimit() + "-" + pdsi.getProbabilityUpperLimit();
}
}
}