Have file updates trigger RSS changes and Subscription actions.

WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java

@@ -186,7 +186,8 @@ public class Erddap extends HttpServlet {
       new ConcurrentHashMap<>(16, 0.75f, 4);
 
   /** The RSS info: key=datasetId, value=utf8 byte[] of rss xml */
-  public final ConcurrentHashMap<String, byte[]> rssHashMap = new ConcurrentHashMap<>(16, 0.75f, 4);
+  public static final ConcurrentHashMap<String, byte[]> rssHashMap =
+      new ConcurrentHashMap<>(16, 0.75f, 4);
 
   public final ConcurrentHashMap<String, int[]> failedLogins =
       new ConcurrentHashMap<>(16, 0.75f, 4);
@@ -23550,7 +23551,8 @@ public void processDataset(EDD dataset, SaxParsingContext context) {
    * @param subject for email messages
    * @param change the change description must be specified or nothing is done
    */
-  protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject, String change) {
+  public static void tryToDoActions(
+      String tDatasetID, EDD cooDataset, String subject, String change) {
     if (String2.isSomething(tDatasetID) && String2.isSomething(change)) {
       if (!String2.isSomething(subject)) subject = "Change to datasetID=" + tDatasetID;
       try {
@@ -23629,7 +23631,7 @@ protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject,
         // trigger RSS action
         // (after new dataset is in place and if there is either a current or older dataset)
         if (cooDataset != null) {
-          cooDataset.updateRSS(this, change);
+          cooDataset.updateRSS(change);
         }
 
       } catch (Throwable subT) {

WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java

@@ -1245,6 +1245,95 @@ public String similar(EDD other) {
     }
   }
 
+  protected Map<String, String> snapshot() {
+    Map<String, String> snapshot = new HashMap<>();
+    snapshot.put("nDv", "" + dataVariables.length);
+
+    for (int dv = 0; dv < dataVariables.length; dv++) {
+      EDV variable = dataVariables()[dv];
+      snapshot.put("dv_" + dv + "_name", variable.destinationName());
+      snapshot.put("dv_" + dv + "_type", variable.destinationDataType());
+      snapshot.put("dv_" + dv + "_attr", variable.combinedAttributes().toString());
+    }
+
+    snapshot.put("globalAttributes", combinedGlobalAttributes.toString());
+
+    return snapshot;
+  }
+
+  /**
+   * This tests if 'oldSnapshot' is different from this in any way. <br>
+   * This test is from the view of a subscriber who wants to know when a dataset has changed in any
+   * way. <br>
+   * So some things like onChange and reloadEveryNMinutes are not checked. <br>
+   * This only lists the first change found.
+   *
+   * <p>EDDGrid overwrites this to also check the axis variables.
+   *
+   * @param old
+   * @return "" if same or message if not.
+   */
+  public String changed(Map<String, String> oldSnapshot) {
+
+    // FUTURE: perhaps it would be nice if EDDTable changed showed new data.
+    //  so it would appear in email subscription and rss.
+    //  but for many datasets (e.g., ndbc met) there are huge number of buoys. so not practical.
+    if (oldSnapshot == null) return EDStatic.EDDChangedWasnt;
+
+    Map<String, String> newSnapshot = snapshot();
+    StringBuilder diff = new StringBuilder();
+    // check most important things first
+    if (!oldSnapshot.get("nDv").equals(newSnapshot.get("nDv"))) {
+      diff.append(
+          MessageFormat.format(
+              EDStatic.EDDChangedDifferentNVar, oldSnapshot.get("nDv"), newSnapshot.get("nDv")));
+      return diff.toString(); // because tests below assume nDv are same
+    }
+
+    int nDv = dataVariables.length;
+    for (int dv = 0; dv < nDv; dv++) {
+      String nameKey = "dv_" + dv + "_name";
+      String typeKey = "dv_" + dv + "_type";
+      String attrKey = "dv_" + dv + "_attr";
+      String msg2 = "#" + dv + "=" + newSnapshot.get(nameKey);
+      if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChanged2Different,
+                    "destinationName",
+                    msg2,
+                    oldSnapshot.get(nameKey),
+                    newSnapshot.get(nameKey))
+                + "\n");
+      }
+      if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChanged2Different,
+                    "destinationDataType",
+                    msg2,
+                    oldSnapshot.get(typeKey),
+                    newSnapshot.get(typeKey))
+                + "\n");
+      }
+      String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey));
+      if (s.length() > 0) {
+        diff.append(
+            MessageFormat.format(EDStatic.EDDChanged1Different, "combinedAttribute", msg2, s)
+                + "\n");
+      }
+    }
+
+    // check least important things last
+    String s =
+        String2.differentLine(
+            oldSnapshot.get("globalAttributes"), newSnapshot.get("globalAttributes"));
+    if (s.length() > 0)
+      diff.append(MessageFormat.format(EDStatic.EDDChangedCGADifferent, s) + "\n");
+
+    return diff.toString();
+  }
+
   //    protected static String test1Changed(String msg, String diff) {
   //        return diff.length() == 0? "" : msg + "\n" + diff + "\n";
   //    }
@@ -1333,7 +1422,7 @@ public String changed(EDD old) {
    *     returns "")
    * @return the rss document
    */
-  public String updateRSS(Erddap erddap, String change) {
+  public String updateRSS(String change) {
     if (change == null || change.length() == 0) return "";
     try {
       // generate the rss xml
@@ -1376,7 +1465,7 @@ public String updateRSS(Erddap erddap, String change) {
 
       // store the xml
       String rssString = rss.toString();
-      if (erddap != null) erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString));
+      Erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString));
       return rssString;
 
     } catch (Throwable rssT) {

WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java

@@ -2613,6 +2613,110 @@ public String similar(EDDGrid other, int firstAxisToMatch, int matchAxisNDigits,
     }
   }
 
+  @Override
+  public Map<String, String> snapshot() {
+    Map<String, String> snapshot = super.snapshot();
+    snapshot.put("nAv", "" + dataVariables.length);
+    for (int av = 0; av < axisVariables.length; av++) {
+      EDVGridAxis variable = axisVariables()[av];
+      snapshot.put("av_" + av + "_name", variable.destinationName());
+      snapshot.put("av_" + av + "_type", variable.destinationDataType());
+      snapshot.put("av_" + av + "_sourceSize", "" + variable.sourceValues().size());
+      snapshot.put("av_" + av + "_minValue", "" + variable.destinationCoarseMin());
+      snapshot.put("av_" + av + "_maxValue", "" + variable.destinationCoarseMax());
+      snapshot.put("av_" + av + "_attr", variable.combinedAttributes().toString());
+    }
+
+    return snapshot;
+  }
+
+  @Override
+  public String changed(Map<String, String> oldSnapshot) {
+    if (oldSnapshot == null) return super.changed(oldSnapshot); // so message is consistent
+
+    Map<String, String> newSnapshot = snapshot();
+    StringBuilder diff = new StringBuilder();
+    // check most important things first
+    if (!oldSnapshot.get("nAv").equals(newSnapshot.get("nAv"))) {
+      diff.append(
+          MessageFormat.format(
+              EDStatic.EDDChangedAxesDifferentNVar,
+              oldSnapshot.get("nAv"),
+              newSnapshot.get("nAv")));
+      return diff.toString(); // because tests below assume nAv are same
+    }
+
+    int nAv = dataVariables.length;
+    for (int av = 0; av < nAv; av++) {
+      String nameKey = "av_" + av + "_name";
+      String typeKey = "av_" + av + "_type";
+      String sourceSizeKey = "av_" + av + "_sourceSize";
+      String minKey = "av_" + av + "_minValue";
+      String maxKey = "av_" + av + "_maxValue";
+      String attrKey = "av_" + av + "_attr";
+      String msg2 = "#" + av + "=" + newSnapshot.get(nameKey);
+      if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChangedAxes2Different,
+                    "destinationName",
+                    msg2,
+                    oldSnapshot.get(nameKey),
+                    newSnapshot.get(nameKey))
+                + "\n");
+      }
+      if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChangedAxes2Different,
+                    "destinationDataType",
+                    msg2,
+                    oldSnapshot.get(typeKey),
+                    newSnapshot.get(typeKey))
+                + "\n");
+      }
+      if (!oldSnapshot.get(sourceSizeKey).equals(newSnapshot.get(sourceSizeKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChangedAxes2Different,
+                    "numberOfValues",
+                    msg2,
+                    oldSnapshot.get(sourceSizeKey),
+                    newSnapshot.get(sourceSizeKey))
+                + "\n");
+      }
+      if (!oldSnapshot.get(minKey).equals(newSnapshot.get(minKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChangedAxes2Different,
+                    "minValue",
+                    msg2,
+                    oldSnapshot.get(minKey),
+                    newSnapshot.get(minKey))
+                + "\n");
+      }
+      if (!oldSnapshot.get(maxKey).equals(newSnapshot.get(maxKey))) {
+        diff.append(
+            MessageFormat.format(
+                    EDStatic.EDDChangedAxes2Different,
+                    "maxValue",
+                    msg2,
+                    oldSnapshot.get(maxKey),
+                    newSnapshot.get(maxKey))
+                + "\n");
+      }
+      String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey));
+      if (s.length() > 0) {
+        diff.append(
+            MessageFormat.format(EDStatic.EDDChangedAxes1Different, "combinedAttribute", msg2, s)
+                + "\n");
+      }
+    }
+
+    diff.append(super.changed(oldSnapshot));
+    return diff.toString();
+  }
+
   /**
    * This tests if 'old' is different from this in any way. <br>
    * This test is from the view of a subscriber who wants to know when a dataset has changed in any

WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java

@@ -44,6 +44,7 @@
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.TimeZone;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -1816,6 +1817,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
       requestReloadASAP();
       return false;
     }
+    Map<String, String> snapshot = snapshot();
 
     // get BadFile and FileTable info and make local copies
     ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file
@@ -2057,14 +2059,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
       }
 
       // after changes all in place
-      // Currently, update() doesn't trigger these changes.
-      // The problem is that some datasets might update every second, others every
-      // day.
-      // Even if they are done, perhaps do them in ERDDAP ((low)update return
-      // changes?)
-      // ?update rss?
-      // ?subscription and onchange actions?
-
+      if (EDStatic.updateSubsRssOnFileChanges) {
+        Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot));
+      }
     }
 
     if (verbose)

WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java

@@ -3210,6 +3210,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
       requestReloadASAP();
       return false;
     }
+    Map<String, String> snapshot = snapshot();
 
     // get BadFile and FileTable info and make local copies
     ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file
@@ -3435,14 +3436,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
       }
 
       // after changes all in place
-      // Currently, update() doesn't trigger these changes.
-      // The problem is that some datasets might update every second, others every
-      // day.
-      // Even if they are done, perhaps do them in ERDDAP ((low)update return
-      // changes?)
-      // ?update rss?
-      // ?subscription and onchange actions?
-
+      if (EDStatic.updateSubsRssOnFileChanges) {
+        Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot));
+      }
     }
 
     if (verbose)

WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java

@@ -809,6 +809,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) {
   public static final boolean variablesMustHaveIoosCategory;
   public static boolean verbose;
   public static boolean useSaxParser;
+  public static boolean updateSubsRssOnFileChanges;
   public static final boolean useEddReflection;
   public static final String[]
       categoryAttributes; // as it appears in metadata (and used for hashmap)
@@ -2294,6 +2295,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) {
       subscriptionSystemActive = getSetupEVBoolean(setup, ev, "subscriptionSystemActive", true);
       convertersActive = getSetupEVBoolean(setup, ev, "convertersActive", true);
       useSaxParser = getSetupEVBoolean(setup, ev, "useSaxParser", false);
+      updateSubsRssOnFileChanges = getSetupEVBoolean(setup, ev, "updateSubsRssOnFileChanges", true);
       useEddReflection = getSetupEVBoolean(setup, ev, "useEddReflection", false);
       slideSorterActive = getSetupEVBoolean(setup, ev, "slideSorterActive", true);
       variablesMustHaveIoosCategory =