From 42805de9c25ea8bf89bf41f430c364cbf9b6a2e6 Mon Sep 17 00:00:00 2001 From: Chris John Date: Fri, 24 Jan 2025 15:38:04 -0500 Subject: [PATCH] Have file updates trigger RSS changes and Subscription actions. --- .../classes/gov/noaa/pfel/erddap/Erddap.java | 8 +- .../gov/noaa/pfel/erddap/dataset/EDD.java | 93 ++++++++++++- .../gov/noaa/pfel/erddap/dataset/EDDGrid.java | 104 ++++++++++++++ .../pfel/erddap/dataset/EDDGridFromFiles.java | 13 +- .../erddap/dataset/EDDTableFromFiles.java | 12 +- .../gov/noaa/pfel/erddap/util/EDStatic.java | 2 + .../dataset/EDDGridFromNcFilesTests.java | 129 +++++++++++++++++- 7 files changed, 338 insertions(+), 23 deletions(-) diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java index 19ea750c..28f6e7ae 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java @@ -186,7 +186,8 @@ public class Erddap extends HttpServlet { new ConcurrentHashMap<>(16, 0.75f, 4); /** The RSS info: key=datasetId, value=utf8 byte[] of rss xml */ - public final ConcurrentHashMap rssHashMap = new ConcurrentHashMap<>(16, 0.75f, 4); + public static final ConcurrentHashMap rssHashMap = + new ConcurrentHashMap<>(16, 0.75f, 4); public final ConcurrentHashMap failedLogins = new ConcurrentHashMap<>(16, 0.75f, 4); @@ -23550,7 +23551,8 @@ public void processDataset(EDD dataset, SaxParsingContext context) { * @param subject for email messages * @param change the change description must be specified or nothing is done */ - protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject, String change) { + public static void tryToDoActions( + String tDatasetID, EDD cooDataset, String subject, String change) { if (String2.isSomething(tDatasetID) && String2.isSomething(change)) { if (!String2.isSomething(subject)) subject = "Change to datasetID=" + tDatasetID; try { @@ -23629,7 +23631,7 @@ protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject, // trigger RSS action // (after new dataset is in place and if there is either a current or older dataset) if (cooDataset != null) { - cooDataset.updateRSS(this, change); + cooDataset.updateRSS(change); } } catch (Throwable subT) { diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java index df3c4424..880e06b9 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java @@ -1245,6 +1245,95 @@ public String similar(EDD other) { } } + protected Map snapshot() { + Map snapshot = new HashMap<>(); + snapshot.put("nDv", "" + dataVariables.length); + + for (int dv = 0; dv < dataVariables.length; dv++) { + EDV variable = dataVariables()[dv]; + snapshot.put("dv_" + dv + "_name", variable.destinationName()); + snapshot.put("dv_" + dv + "_type", variable.destinationDataType()); + snapshot.put("dv_" + dv + "_attr", variable.combinedAttributes().toString()); + } + + snapshot.put("globalAttributes", combinedGlobalAttributes.toString()); + + return snapshot; + } + + /** + * This tests if 'oldSnapshot' is different from this in any way.
+ * This test is from the view of a subscriber who wants to know when a dataset has changed in any + * way.
+ * So some things like onChange and reloadEveryNMinutes are not checked.
+ * This only lists the first change found. + * + *

EDDGrid overwrites this to also check the axis variables. + * + * @param old + * @return "" if same or message if not. + */ + public String changed(Map oldSnapshot) { + + // FUTURE: perhaps it would be nice if EDDTable changed showed new data. + // so it would appear in email subscription and rss. + // but for many datasets (e.g., ndbc met) there are huge number of buoys. so not practical. + if (oldSnapshot == null) return EDStatic.EDDChangedWasnt; + + Map newSnapshot = snapshot(); + StringBuilder diff = new StringBuilder(); + // check most important things first + if (!oldSnapshot.get("nDv").equals(newSnapshot.get("nDv"))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedDifferentNVar, oldSnapshot.get("nDv"), newSnapshot.get("nDv"))); + return diff.toString(); // because tests below assume nDv are same + } + + int nDv = dataVariables.length; + for (int dv = 0; dv < nDv; dv++) { + String nameKey = "dv_" + dv + "_name"; + String typeKey = "dv_" + dv + "_type"; + String attrKey = "dv_" + dv + "_attr"; + String msg2 = "#" + dv + "=" + newSnapshot.get(nameKey); + if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChanged2Different, + "destinationName", + msg2, + oldSnapshot.get(nameKey), + newSnapshot.get(nameKey)) + + "\n"); + } + if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChanged2Different, + "destinationDataType", + msg2, + oldSnapshot.get(typeKey), + newSnapshot.get(typeKey)) + + "\n"); + } + String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey)); + if (s.length() > 0) { + diff.append( + MessageFormat.format(EDStatic.EDDChanged1Different, "combinedAttribute", msg2, s) + + "\n"); + } + } + + // check least important things last + String s = + String2.differentLine( + oldSnapshot.get("globalAttributes"), newSnapshot.get("globalAttributes")); + if (s.length() > 0) + diff.append(MessageFormat.format(EDStatic.EDDChangedCGADifferent, s) + "\n"); + + return diff.toString(); + } + // protected static String test1Changed(String msg, String diff) { // return diff.length() == 0? "" : msg + "\n" + diff + "\n"; // } @@ -1333,7 +1422,7 @@ public String changed(EDD old) { * returns "") * @return the rss document */ - public String updateRSS(Erddap erddap, String change) { + public String updateRSS(String change) { if (change == null || change.length() == 0) return ""; try { // generate the rss xml @@ -1376,7 +1465,7 @@ public String updateRSS(Erddap erddap, String change) { // store the xml String rssString = rss.toString(); - if (erddap != null) erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString)); + Erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString)); return rssString; } catch (Throwable rssT) { diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java index 523586a6..6151b149 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java @@ -2613,6 +2613,110 @@ public String similar(EDDGrid other, int firstAxisToMatch, int matchAxisNDigits, } } + @Override + public Map snapshot() { + Map snapshot = super.snapshot(); + snapshot.put("nAv", "" + dataVariables.length); + for (int av = 0; av < axisVariables.length; av++) { + EDVGridAxis variable = axisVariables()[av]; + snapshot.put("av_" + av + "_name", variable.destinationName()); + snapshot.put("av_" + av + "_type", variable.destinationDataType()); + snapshot.put("av_" + av + "_sourceSize", "" + variable.sourceValues().size()); + snapshot.put("av_" + av + "_minValue", "" + variable.destinationCoarseMin()); + snapshot.put("av_" + av + "_maxValue", "" + variable.destinationCoarseMax()); + snapshot.put("av_" + av + "_attr", variable.combinedAttributes().toString()); + } + + return snapshot; + } + + @Override + public String changed(Map oldSnapshot) { + if (oldSnapshot == null) return super.changed(oldSnapshot); // so message is consistent + + Map newSnapshot = snapshot(); + StringBuilder diff = new StringBuilder(); + // check most important things first + if (!oldSnapshot.get("nAv").equals(newSnapshot.get("nAv"))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxesDifferentNVar, + oldSnapshot.get("nAv"), + newSnapshot.get("nAv"))); + return diff.toString(); // because tests below assume nAv are same + } + + int nAv = dataVariables.length; + for (int av = 0; av < nAv; av++) { + String nameKey = "av_" + av + "_name"; + String typeKey = "av_" + av + "_type"; + String sourceSizeKey = "av_" + av + "_sourceSize"; + String minKey = "av_" + av + "_minValue"; + String maxKey = "av_" + av + "_maxValue"; + String attrKey = "av_" + av + "_attr"; + String msg2 = "#" + av + "=" + newSnapshot.get(nameKey); + if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxes2Different, + "destinationName", + msg2, + oldSnapshot.get(nameKey), + newSnapshot.get(nameKey)) + + "\n"); + } + if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxes2Different, + "destinationDataType", + msg2, + oldSnapshot.get(typeKey), + newSnapshot.get(typeKey)) + + "\n"); + } + if (!oldSnapshot.get(sourceSizeKey).equals(newSnapshot.get(sourceSizeKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxes2Different, + "numberOfValues", + msg2, + oldSnapshot.get(sourceSizeKey), + newSnapshot.get(sourceSizeKey)) + + "\n"); + } + if (!oldSnapshot.get(minKey).equals(newSnapshot.get(minKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxes2Different, + "minValue", + msg2, + oldSnapshot.get(minKey), + newSnapshot.get(minKey)) + + "\n"); + } + if (!oldSnapshot.get(maxKey).equals(newSnapshot.get(maxKey))) { + diff.append( + MessageFormat.format( + EDStatic.EDDChangedAxes2Different, + "maxValue", + msg2, + oldSnapshot.get(maxKey), + newSnapshot.get(maxKey)) + + "\n"); + } + String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey)); + if (s.length() > 0) { + diff.append( + MessageFormat.format(EDStatic.EDDChangedAxes1Different, "combinedAttribute", msg2, s) + + "\n"); + } + } + + diff.append(super.changed(oldSnapshot)); + return diff.toString(); + } + /** * This tests if 'old' is different from this in any way.
* This test is from the view of a subscriber who wants to know when a dataset has changed in any diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java index 9c833523..43929012 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.HashSet; +import java.util.Map; import java.util.TimeZone; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -1816,6 +1817,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr requestReloadASAP(); return false; } + Map snapshot = snapshot(); // get BadFile and FileTable info and make local copies ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file @@ -2057,14 +2059,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr } // after changes all in place - // Currently, update() doesn't trigger these changes. - // The problem is that some datasets might update every second, others every - // day. - // Even if they are done, perhaps do them in ERDDAP ((low)update return - // changes?) - // ?update rss? - // ?subscription and onchange actions? - + if (EDStatic.updateSubsRssOnFileChanges) { + Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot)); + } } if (verbose) diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java index 57a99f36..c22dba28 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java @@ -3210,6 +3210,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr requestReloadASAP(); return false; } + Map snapshot = snapshot(); // get BadFile and FileTable info and make local copies ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file @@ -3435,14 +3436,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr } // after changes all in place - // Currently, update() doesn't trigger these changes. - // The problem is that some datasets might update every second, others every - // day. - // Even if they are done, perhaps do them in ERDDAP ((low)update return - // changes?) - // ?update rss? - // ?subscription and onchange actions? - + if (EDStatic.updateSubsRssOnFileChanges) { + Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot)); + } } if (verbose) diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java index d614582c..b46d83d9 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java @@ -809,6 +809,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { public static final boolean variablesMustHaveIoosCategory; public static boolean verbose; public static boolean useSaxParser; + public static boolean updateSubsRssOnFileChanges; public static final boolean useEddReflection; public static final String[] categoryAttributes; // as it appears in metadata (and used for hashmap) @@ -2294,6 +2295,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { subscriptionSystemActive = getSetupEVBoolean(setup, ev, "subscriptionSystemActive", true); convertersActive = getSetupEVBoolean(setup, ev, "convertersActive", true); useSaxParser = getSetupEVBoolean(setup, ev, "useSaxParser", false); + updateSubsRssOnFileChanges = getSetupEVBoolean(setup, ev, "updateSubsRssOnFileChanges", true); useEddReflection = getSetupEVBoolean(setup, ev, "useEddReflection", false); slideSorterActive = getSetupEVBoolean(setup, ev, "slideSorterActive", true); variablesMustHaveIoosCategory = diff --git a/src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFilesTests.java b/src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFilesTests.java index 8acb176f..9ba8d903 100644 --- a/src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFilesTests.java +++ b/src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFilesTests.java @@ -13,10 +13,12 @@ import gov.noaa.pfel.coastwatch.sgt.SgtMap; import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; import gov.noaa.pfel.coastwatch.util.SSR; +import gov.noaa.pfel.erddap.Erddap; import gov.noaa.pfel.erddap.GenerateDatasetsXml; import gov.noaa.pfel.erddap.util.EDStatic; import java.nio.file.Path; import java.util.Arrays; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -37,9 +39,17 @@ import ucar.nc2.dataset.NetcdfDatasets; class EDDGridFromNcFilesTests { + private static boolean initialUpdateRss; + @BeforeAll static void init() { Initialization.edStatic(); + initialUpdateRss = EDStatic.updateSubsRssOnFileChanges; + } + + @AfterEach + void cleanup() { + EDStatic.updateSubsRssOnFileChanges = initialUpdateRss; } /** This prints time, lat, and lon values from an .ncml dataset. */ @@ -14049,10 +14059,12 @@ void testRTechHdf() throws Throwable { * * @throws Throwable if trouble */ - @org.junit.jupiter.api.Test @TagSlowTests - void testUpdate() throws Throwable { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testUpdate(boolean allowRssUpdates) throws Throwable { // String2.log("\n*** EDDGridFromNcFiles.testUpdate()\n"); + String datasetId = "testGriddedNcFiles"; EDDGridFromNcFiles eddGrid = (EDDGridFromNcFiles) EDDTestDataset.gettestGriddedNcFiles(); String dataDir = eddGrid.fileDir; String tDir = EDStatic.fullTestCacheDirectory; @@ -14060,6 +14072,9 @@ void testUpdate() throws Throwable { String dataQuery = "x_wind[][][100][100]"; String tName, results, expected; int language = 0; + // This is set back to the initial value in the @After for the class. + EDStatic.updateSubsRssOnFileChanges = allowRssUpdates; + Erddap.rssHashMap.remove(datasetId); // *** read the original data String2.log("\n*** read original data\n"); @@ -14126,6 +14141,10 @@ void testUpdate() throws Throwable { oldMaxTime, "time_coverage_end"); + byte[] rssAr = Erddap.rssHashMap.get(datasetId); + String rss = String2.utf8BytesToString(rssAr); + Test.ensureEqual(rss, null, "initial_rss"); + // *** rename a data file so it doesn't match regex try { String2.log("\n*** rename a data file so it doesn't match regex\n"); @@ -14185,6 +14204,46 @@ void testUpdate() throws Throwable { oldMaxTime, "time_coverage_end"); + rssAr = Erddap.rssHashMap.get(datasetId); + rss = String2.utf8BytesToString(rssAr); + if (!allowRssUpdates) { + Test.ensureEqual(rss, null, "initial_rss"); + } else { + String rssExpected = + "\n" + + "\n" + + " \n" + + " ERDDAP: Wind, QuikSCAT, Global, Science Quality (1 Day Composite)\n" + + " This RSS feed changes when the dataset changes.\n" + + " &erddapUrl;/griddap/testGriddedNcFiles.html\n" + + " PUBLISHED_DATE\n" + + " \n" + + " This dataset changed YYYY-MM-DDThh:mm:ssZ\n" + + " &erddapUrl;/griddap/testGriddedNcFiles.html\n" + + " The numberOfValues for axisVariable #0=time changed:\n" + + " old=10,\n" + + " new=7.\n" + + "The minValue for axisVariable #0=time changed:\n" + + " old=1.1991456E9,\n" + + " new=1.1994048E9.\n" + + "The combinedAttribute for axisVariable #0=time changed:\n" + + " old line #2=" actual_range=1.1991888E9d,1.1999664E9d",\n" + + " new line #2=" actual_range=1.199448E9d,1.1999664E9d".\n" + + "A combinedGlobalAttribute changed:\n" + + " old line #47=" time_coverage_start=2008-01-01T12:00:00Z",\n" + + " new line #47=" time_coverage_start=2008-01-04T12:00:00Z".\n" + + "\n" + + " \n" + + " \n" + + "\n"; + rss = rss.replaceAll(".*", "PUBLISHED_DATE"); + rss = + rss.replaceAll( + "This dataset changed ....-..-..T..:..:..Z", + "This dataset changed YYYY-MM-DDThh:mm:ssZ"); + Test.ensureEqual(rss, rssExpected, "results=\n" + rss); + } + } finally { // rename it back to original String2.log("\n*** rename it back to original\n"); @@ -14226,6 +14285,46 @@ void testUpdate() throws Throwable { oldMaxTime, "time_coverage_end"); + rssAr = Erddap.rssHashMap.get(datasetId); + rss = String2.utf8BytesToString(rssAr); + String rssExpected = + "\n" + + "\n" + + " \n" + + " ERDDAP: Wind, QuikSCAT, Global, Science Quality (1 Day Composite)\n" + + " This RSS feed changes when the dataset changes.\n" + + " &erddapUrl;/griddap/testGriddedNcFiles.html\n" + + " PUBLISHED_DATE\n" + + " \n" + + " This dataset changed YYYY-MM-DDThh:mm:ssZ\n" + + " &erddapUrl;/griddap/testGriddedNcFiles.html\n" + + " The numberOfValues for axisVariable #0=time changed:\n" + + " old=7,\n" + + " new=10.\n" + + "The minValue for axisVariable #0=time changed:\n" + + " old=1.1994048E9,\n" + + " new=1.1991456E9.\n" + + "The combinedAttribute for axisVariable #0=time changed:\n" + + " old line #2=" actual_range=1.199448E9d,1.1999664E9d",\n" + + " new line #2=" actual_range=1.1991888E9d,1.1999664E9d".\n" + + "A combinedGlobalAttribute changed:\n" + + " old line #47=" time_coverage_start=2008-01-04T12:00:00Z",\n" + + " new line #47=" time_coverage_start=2008-01-01T12:00:00Z".\n" + + "\n" + + " \n" + + " \n" + + "\n"; + if (!allowRssUpdates) { + Test.ensureEqual(rss, null, "initial_rss"); + } else { + rss = rss.replaceAll(".*", "PUBLISHED_DATE"); + rss = + rss.replaceAll( + "This dataset changed ....-..-..T..:..:..Z", + "This dataset changed YYYY-MM-DDThh:mm:ssZ"); + Test.ensureEqual(rss, rssExpected, "results=\n" + rss); + } + // *** rename a non-data file so it matches the regex try { String2.log("\n*** rename an invalid file to be a valid name\n"); @@ -14269,6 +14368,19 @@ void testUpdate() throws Throwable { eddGrid.combinedGlobalAttributes().getString("time_coverage_end"), oldMaxTime, "time_coverage_end"); + + rssAr = Erddap.rssHashMap.get(datasetId); + rss = String2.utf8BytesToString(rssAr); + if (!allowRssUpdates) { + Test.ensureEqual(rss, null, "initial_rss"); + } else { + rss = rss.replaceAll(".*", "PUBLISHED_DATE"); + rss = + rss.replaceAll( + "This dataset changed ....-..-..T..:..:..Z", + "This dataset changed YYYY-MM-DDThh:mm:ssZ"); + Test.ensureEqual(rss, rssExpected, "results=\n" + rss); + } } catch (Exception e) { String2.log("Note exception being thrown:\n" + MustBe.throwableToString(e)); throw e; @@ -14314,6 +14426,19 @@ void testUpdate() throws Throwable { oldMaxTime, "time_coverage_end"); + rssAr = Erddap.rssHashMap.get(datasetId); + rss = String2.utf8BytesToString(rssAr); + if (!allowRssUpdates) { + Test.ensureEqual(rss, null, "initial_rss"); + } else { + rss = rss.replaceAll(".*", "PUBLISHED_DATE"); + rss = + rss.replaceAll( + "This dataset changed ....-..-..T..:..:..Z", + "This dataset changed YYYY-MM-DDThh:mm:ssZ"); + Test.ensureEqual(rss, rssExpected, "results=\n" + rss); + } + // test time for update if 0 events long cumTime = 0; for (int i = 0; i < 1000; i++) {