Log parser time mean

Framework/API/test/LogManagerTest.h

@@ -359,7 +359,7 @@ class LogManagerTest : public CxxTest::TestSuite {
     // all values were calculated using a independent implementation in python
     const double FIRST_VALUE{2.}; // also the min
     const double LAST_VALUE{24.}; // also the max
-    const double TIME_AVG_MEAN{15.357142857142858};
+    const double TIME_AVG_MEAN{18.2380952348};
     // const double TIME_AVG_STDDEV {8.523975789812294};
 
     TS_ASSERT_EQUALS(runInfo.getProperty(name)->size(), 10);
@@ -371,7 +371,7 @@ class LogManagerTest : public CxxTest::TestSuite {
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::LastValue), LAST_VALUE, 1e-12);
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::Median), 13.0, 1e-12);
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::StdDev), 9.1104335791443, 1e-12);
-    TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAveragedMean), TIME_AVG_MEAN, 1e-12);
+    TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAveragedMean), TIME_AVG_MEAN, 1e-8);
     // TODO not ready
     // TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAverageStdDev), TIME_AVG_STDDEV, 1e-12);
 
@@ -389,7 +389,7 @@ class LogManagerTest : public CxxTest::TestSuite {
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::LastValue), LAST_VALUE, 1e-12);     // TODO
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::Median), 6.0, 1e-12);               // TODO
     TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::StdDev), 8.937367800973425, 1e-12); // TODO
-    TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAveragedMean), TIME_AVG_MEAN, 1e-12);
+    TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAveragedMean), TIME_AVG_MEAN, 1e-8);
     // TODO not ready
     // TS_ASSERT_DELTA(runInfo.getPropertyAsSingleValue(name, Math::TimeAverageStdDev), TIME_AVG_STDDEV, 1e-12);
   }
@@ -425,7 +425,7 @@ class LogManagerTest : public CxxTest::TestSuite {
     const std::string name = "series";
     addTestTimeSeries<double>(run, name);
 
-    TS_ASSERT_DELTA(run.getTimeAveragedStd(name), 8.5239, 0.001);
+    TS_ASSERT_DELTA(run.getTimeAveragedStd(name), 8.0646, 0.001);
   }
 
   void test_getStatistics() {
@@ -441,7 +441,7 @@ class LogManagerTest : public CxxTest::TestSuite {
       TS_ASSERT_DELTA(stats.standard_deviation, 0.0, 0.001);
       TS_ASSERT_DELTA(stats.time_mean, value, 0.001);
       TS_ASSERT_DELTA(stats.time_standard_deviation, 0.0, 0.001);
-      TS_ASSERT(isnan(stats.duration));
+      TS_ASSERT(std::isnan(stats.duration));
     };
 
     // test valid single value property
@@ -458,13 +458,13 @@ class LogManagerTest : public CxxTest::TestSuite {
     {
       addTestPropertyWithValue<std::string>(run, "single-string", "46");
       auto stats = run.getStatistics("single-string");
-      TS_ASSERT_EQUALS(isnan(stats.minimum), true);
-      TS_ASSERT_EQUALS(isnan(stats.maximum), true);
-      TS_ASSERT_EQUALS(isnan(stats.mean), true);
-      TS_ASSERT_EQUALS(isnan(stats.standard_deviation), true);
-      TS_ASSERT_EQUALS(isnan(stats.time_mean), true);
-      TS_ASSERT_EQUALS(isnan(stats.time_standard_deviation), true);
-      TS_ASSERT_EQUALS(isnan(stats.duration), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.minimum), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.maximum), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.mean), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.standard_deviation), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.time_mean), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.time_standard_deviation), true);
+      TS_ASSERT_EQUALS(std::isnan(stats.duration), true);
     }
 
     // test time series
@@ -480,9 +480,9 @@ class LogManagerTest : public CxxTest::TestSuite {
       TS_ASSERT_DELTA(stats.mean, 13.0, 0.001);
       TS_ASSERT_DELTA(stats.median, 13.0, 0.001);
       TS_ASSERT_DELTA(stats.standard_deviation, 9.1104, 0.001);
-      TS_ASSERT_DELTA(stats.time_mean, 15.3571, 0.001);
-      TS_ASSERT_DELTA(stats.time_standard_deviation, 8.5239, 0.001);
-      TS_ASSERT_DELTA(stats.duration, 140.0, 0.001);
+      TS_ASSERT_DELTA(stats.time_mean, 18.2381, 0.001);
+      TS_ASSERT_DELTA(stats.time_standard_deviation, 8.06464, 0.001);
+      TS_ASSERT_DELTA(stats.duration, 210.0, 0.001);
     }
   }
 

Framework/Kernel/inc/MantidKernel/ITimeSeriesProperty.h

@@ -44,13 +44,11 @@ class ITimeSeriesProperty {
   virtual std::pair<double, double> averageAndStdDevInFilter(const std::vector<SplittingInterval> &filter) const = 0;
   /// Return the time series's times as a vector<DateAndTime>
   virtual std::vector<Types::Core::DateAndTime> timesAsVector() const = 0;
-  /// Returns the calculated time weighted average value
-  virtual double timeAverageValue() const = 0;
   /** Returns the calculated time weighted average value.
    * @param timeRoi  Object that holds information about when the time measurement was active.
    * @return The time-weighted average value of the log when the time measurement was active.
    */
-  virtual double timeAverageValue(const TimeROI &timeRoi) const = 0;
+  virtual double timeAverageValue(const TimeROI *timeRoi = nullptr) const = 0;
   /// Return a TimeSeriesPropertyStatistics object
   virtual TimeSeriesPropertyStatistics getStatistics(const TimeROI *roi = nullptr) const = 0;
   /// Returns the real size of the time series property map:

Framework/Kernel/inc/MantidKernel/LogParser.h

@@ -29,6 +29,7 @@ namespace Kernel {
 //-------------------------------------------------------------------------
 // Forward declarations
 //-------------------------------------------------------------------------
+class TimeROI;
 class Property;
 template <typename T> class TimeSeriesProperty;
 
@@ -110,7 +111,7 @@ class MANTID_KERNEL_DLL LogParser {
 };
 
 /// Returns the mean value if the property is TimeSeriesProperty<double>
-MANTID_KERNEL_DLL double timeMean(const Kernel::Property *p);
+MANTID_KERNEL_DLL double timeMean(const Kernel::Property *p, const TimeROI *roi = nullptr);
 
 } // namespace Kernel
 } // namespace Mantid

Framework/Kernel/inc/MantidKernel/TimeSeriesProperty.h

@@ -199,15 +199,15 @@ template <typename TYPE> class DLLExport TimeSeriesProperty : public Property, p
   double averageValueInFilter(const std::vector<SplittingInterval> &filter) const override;
   /// @copydoc Mantid::Kernel::ITimeSeriesProperty::averageAndStdDevInFilter()
   std::pair<double, double> averageAndStdDevInFilter(const std::vector<SplittingInterval> &filter) const override;
-  /// @copydoc Mantid::Kernel::ITimeSeriesProperty::timeAverageValue()
-  /// Time weighted mean and standard deviation
+  /** Returns the calculated time weighted mean and standard deviation values.
+   * @return The time-weighted average value of the log when the time measurement was active.
+   */
   std::pair<double, double> timeAverageValueAndStdDev() const;
-  double timeAverageValue() const override;
   /** Returns the calculated time weighted average value.
    * @param timeRoi  Object that holds information about when the time measurement was active.
    * @return The time-weighted average value of the log when the time measurement was active.
    */
-  double timeAverageValue(const TimeROI &timeRoi) const override;
+  double timeAverageValue(const TimeROI *timeRoi = nullptr) const override;
   /// generate constant time-step histogram from the property values
   void histogramData(const Types::Core::DateAndTime &tMin, const Types::Core::DateAndTime &tMax,
                      std::vector<double> &counts) const;

Framework/Kernel/src/LogParser.cpp

@@ -11,6 +11,7 @@
 #include "MantidKernel/Logger.h"
 #include "MantidKernel/PropertyWithValue.h"
 #include "MantidKernel/Strings.h"
+#include "MantidKernel/TimeROI.h"
 #include "MantidKernel/TimeSeriesProperty.h"
 
 #include <fstream>
@@ -306,40 +307,18 @@ bool LogParser::isICPEventLogNewStyle(const std::multimap<Types::Core::DateAndTi
  *
  * @param p :: Property with the data. Will throw if not
  *             TimeSeriesProperty<double>.
+ * @param roi :: TimeROI for when the log is being used
  * @return The mean value over time.
  * @throw runtime_error if the property is not TimeSeriesProperty<double>
  */
-double timeMean(const Kernel::Property *p) {
+double timeMean(const Kernel::Property *p, const Kernel::TimeROI *roi) {
   const auto *dp = dynamic_cast<const Kernel::TimeSeriesProperty<double> *>(p);
   if (!dp) {
     throw std::runtime_error("Property of a wrong type. Cannot be cast to a "
                              "TimeSeriesProperty<double>.");
   }
 
-  // Special case for only one value - the algorithm
-  if (dp->size() == 1) {
-    return dp->nthValue(1);
-  }
-  double res = 0.;
-  Types::Core::time_duration total(0, 0, 0, 0);
-  size_t dp_size = dp->size();
-  for (size_t i = 0; i < dp_size; i++) {
-    Kernel::TimeInterval t = dp->nthInterval(static_cast<int>(i));
-    Types::Core::time_duration dt = t.length();
-    total += dt;
-    res += dp->nthValue(static_cast<int>(i)) * Types::Core::DateAndTime::secondsFromDuration(dt);
-  }
-
-  double total_seconds = Types::Core::DateAndTime::secondsFromDuration(total);
-
-  // If all the time stamps were the same, just return the first value.
-  if (total_seconds == 0.0)
-    res = dp->nthValue(1);
-
-  if (total_seconds > 0)
-    res /= total_seconds;
-
-  return res;
+  return dp->timeAverageValue(roi);
 }
 
 } // namespace Kernel

Framework/Kernel/src/TimeSeriesProperty.cpp

@@ -788,30 +788,20 @@ void TimeSeriesProperty<std::string>::expandFilterToRange(std::vector<SplittingI
                                        "properties");
 }
 
-/** Calculates the time-weighted average of a property.
- *  @return The time-weighted average value of the log.
- */
-template <typename TYPE> double TimeSeriesProperty<TYPE>::timeAverageValue() const {
-  double retVal = 0.0;
-  try {
-    const auto &filter = getSplittingIntervals();
-    retVal = this->averageValueInFilter(filter);
-  } catch (std::exception &) {
-    // just return nan
-    retVal = std::numeric_limits<double>::quiet_NaN();
-  }
-  return retVal;
-}
-
 /** Returns the calculated time weighted average value.
  * @param timeRoi  Object that holds information about when the time measurement was active.
  * @return The time-weighted average value of the log when the time measurement was active.
  */
-template <typename TYPE> double TimeSeriesProperty<TYPE>::timeAverageValue(const TimeROI &timeRoi) const {
+template <typename TYPE> double TimeSeriesProperty<TYPE>::timeAverageValue(const TimeROI *timeRoi) const {
   double retVal = 0.0;
   try {
-    const auto &filter = timeRoi.toSplitters();
-    retVal = this->averageValueInFilter(filter);
+    if ((timeRoi == nullptr) || (timeRoi->empty())) {
+      const auto &filter = getSplittingIntervals();
+      retVal = this->averageValueInFilter(filter);
+    } else {
+      const auto &filter = timeRoi->toSplitters();
+      retVal = this->averageValueInFilter(filter);
+    }
   } catch (std::exception &) {
     // just return nan
     retVal = std::numeric_limits<double>::quiet_NaN();
@@ -829,16 +819,16 @@ template <typename TYPE>
 double TimeSeriesProperty<TYPE>::averageValueInFilter(const std::vector<SplittingInterval> &filter) const {
   // TODO: Consider logs that aren't giving starting values.
 
+  // If there's just a single value in the log, return that.
+  if (size() == 1) {
+    return static_cast<double>(this->firstValue());
+  }
+
   // First of all, if the log or the filter is empty, return NaN
   if (realSize() == 0 || filter.empty()) {
     return std::numeric_limits<double>::quiet_NaN();
   }
 
-  // If there's just a single value in the log, return that.
-  if (realSize() == 1) {
-    return static_cast<double>(m_values.front().value());
-  }
-
   sortIfNecessary();
 
   double numerator(0.0), totalTime(0.0);
@@ -863,8 +853,14 @@ double TimeSeriesProperty<TYPE>::averageValueInFilter(const std::vector<Splittin
     numerator += DateAndTime::secondsFromDuration(time.end() - startTime) * value;
   }
 
-  // 'Normalise' by the total time
-  return numerator / totalTime;
+  if (totalTime > 0) {
+    // 'Normalise' by the total time
+    return numerator / totalTime;
+  } else {
+    // give simple mean
+    const auto stats = Mantid::Kernel::getStatistics(this->valuesAsVector(), Mantid::Kernel::Math::StatisticType::Mean);
+    return stats.mean;
+  }
 }
 
 /** Function specialization for TimeSeriesProperty<std::string>
@@ -1551,10 +1547,19 @@ template <typename TYPE> TimeInterval TimeSeriesProperty<TYPE>::nthInterval(int
       ;
     } else if (n == static_cast<int>(m_values.size()) - 1) {
       // 2. Last one by making up an end time.
-      time_duration d = m_values.rbegin()->time() - (m_values.rbegin() + 1)->time();
-      DateAndTime endTime = m_values.rbegin()->time() + d;
-      Kernel::TimeInterval dt(m_values.rbegin()->time(), endTime);
-      deltaT = dt;
+      // the last time is the last thing known
+      const auto ultimate = m_values.rbegin()->time();
+      // go backwards from the time before it that is different
+      int counter = 0;
+      while (DateAndTime::secondsFromDuration(ultimate - (m_values.rbegin() + counter)->time()) == 0.) {
+        counter += 1;
+      }
+      // get the last time that is different
+      time_duration lastDuration = m_values.rbegin()->time() - (m_values.rbegin() + counter)->time();
+      // the last duration is equal to the previous, non-zero, duration
+      DateAndTime endTime = m_values.rbegin()->time() + lastDuration;
+
+      deltaT = Kernel::TimeInterval(m_values.rbegin()->time(), endTime);
     } else {
       // 3. Regular
       DateAndTime startT = m_values[static_cast<std::size_t>(n)].time();
@@ -2391,7 +2396,13 @@ template <typename TYPE> std::vector<SplittingInterval> TimeSeriesProperty<TYPE>
   std::vector<SplittingInterval> intervals;
   // Case where there is no filter
   if (m_filter.empty()) {
-    intervals.emplace_back(firstTime(), lastTime());
+    // interval calculates what a reasonable place to put the end point for the last log entry is
+    // this *should* be a reasonable estimate and *is* better than always, effectively, ignoring
+    // the last log value. The value is different than that from lastTime()
+    auto lastInterval = this->nthInterval(this->size() - 1);
+
+    intervals.emplace_back(firstTime(), lastInterval.end());
+
     return intervals;
   }
 

Framework/Kernel/test/TimeSeriesPropertyTest.h

@@ -672,14 +672,13 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite {
   }
 
   void test_timeAverageValue() {
+    // values are equally spaced
     auto dblLog = createDoubleTSP();
     auto intLog = createIntegerTSP(5);
 
     // average values
-    const double dblMean = dblLog->timeAverageValue();
-    TS_ASSERT_DELTA(dblMean, 7.6966, .0001);
-    const double intMean = intLog->timeAverageValue();
-    TS_ASSERT_DELTA(intMean, 2.5, .0001);
+    TS_ASSERT_DELTA(dblLog->timeAverageValue(), dblLog->mean(), .0001);
+    TS_ASSERT_DELTA(intLog->timeAverageValue(), intLog->mean(), .0001);
 
     // Clean up
     delete dblLog;
@@ -689,7 +688,7 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite {
   void test_timeAverageValueWithROI() {
     auto dblLog = createDoubleTSP();
     TimeROI *rois = createTimeRoi();
-    const double dblMean = dblLog->timeAverageValue(*rois);
+    const double dblMean = dblLog->timeAverageValue(rois);
     delete dblLog; // clean up
     delete rois;
     const double expected = (5.0 * 9.99 + 5.0 * 7.55 + 5.0 * 5.55 + 5.0 * 10.55) / (5.0 + 5.0 + 5.0 + 5.0);
@@ -1096,11 +1095,11 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite {
     TS_ASSERT_DELTA(stats.maximum, 11.0, 1e-3);
     TS_ASSERT_DELTA(stats.median, 6.0, 1e-3);
     TS_ASSERT_DELTA(stats.mean, 6.0, 1e-3);
-    TS_ASSERT_DELTA(stats.duration, 100.0, 1e-3);
+    TS_ASSERT_DELTA(stats.duration, 110.0, 1e-3);
     TS_ASSERT_DELTA(stats.standard_deviation, 3.1622, 1e-3);
-    TS_ASSERT_DELTA(log->timeAverageValue(), 5.5, 1e-3);
-    TS_ASSERT_DELTA(stats.time_mean, 5.5, 1e-3);
-    TS_ASSERT_DELTA(stats.time_standard_deviation, 2.872, 1e-3);
+    TS_ASSERT_DELTA(log->timeAverageValue(), stats.mean, 1e-3);
+    TS_ASSERT_DELTA(stats.time_mean, stats.mean, 1e-3);
+    TS_ASSERT_DELTA(stats.time_standard_deviation, stats.standard_deviation, 1e-3);
 
     delete log;
   }
@@ -2262,7 +2261,7 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite {
   void test_filterByTime_out_of_range_filters_nothing() {
     TimeSeriesProperty<int> *log = createIntegerTSP(6);
 
-    size_t original_size = log->realSize();
+    size_t original_size = std::size_t(log->realSize());
 
     TS_ASSERT_EQUALS(original_size, 6);
 
@@ -2313,7 +2312,12 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite {
     TS_ASSERT_EQUALS(intervals.size(), 1);
     const auto &range = intervals.front();
     TS_ASSERT_EQUALS(range.begin(), log->firstTime());
-    TS_ASSERT_EQUALS(range.end(), log->lastTime());
+
+    // the range is extended by the last difference in times
+    // this is to make the last value count as much as the penultimate
+    const auto lastDuration = log->nthInterval(log->size() - 1).length();
+    const auto stop = log->lastTime() + lastDuration;
+    TS_ASSERT_EQUALS(range.end(), stop);
   }
 
   void test_getSplittingIntervals_repeatedEntries() {

Framework/MDAlgorithms/test/ConvertToMDMinMaxLocalTest.h

@@ -184,8 +184,8 @@ class ConvertToMDMinMaxLocalTest : public CxxTest::TestSuite {
     TS_ASSERT_THROWS_NOTHING(alg.execute(););
     TS_ASSERT(alg.isExecuted());
     // Check the results
-    TS_ASSERT_EQUALS(alg.getPropertyValue("MinValues"), "0.12187,7.69667");
-    TS_ASSERT_EQUALS(alg.getPropertyValue("MaxValues"), "0.126745,7.69667");
+    TS_ASSERT_EQUALS(alg.getPropertyValue("MinValues"), "0.12187,8.41");
+    TS_ASSERT_EQUALS(alg.getPropertyValue("MaxValues"), "0.126745,8.41");
     // Remove workspace from the data service.
     Mantid::API::AnalysisDataService::Instance().remove(WSName);
   }
@@ -232,7 +232,7 @@ class ConvertToMDMinMaxLocalTest : public CxxTest::TestSuite {
     ws->mutableSample().setOrientedLattice(std::make_unique<Mantid::Geometry::OrientedLattice>(2, 3, 4, 90, 90, 90));
 
     // time average value of this is the simple average
-    // of the first three values = 7.69667
+    // of the values = 8.41
     Mantid::Kernel::TimeSeriesProperty<double> *p = new Mantid::Kernel::TimeSeriesProperty<double>("doubleProp");
     TS_ASSERT_THROWS_NOTHING(p->addValue("2007-11-30T16:17:00", 9.99));
     TS_ASSERT_THROWS_NOTHING(p->addValue("2007-11-30T16:17:10", 7.55));