PyPersistTest now compares dat and ref YAML tree entry by entry, allowing for...

PyPersistTest now compares dat and ref YAML tree entry by entry, allowing for a tolerance for floating-point numbers.
Still to do:
- allow for "VARIABILITY" anywhere in the tree
- generate YAML output from Fit tests.

Tests/Functional/PyCore/persistence/PyPersistenceTest.cpp

@@ -21,12 +21,15 @@
 #include "IntensityDataIOFactory.h"
 #include "PythonFormatting.h"
 #include "SimulationFactory.h"
+#include "Numeric.h"
 #include "Utils.h"
+#include <yaml-cpp/yaml.h>
 #include <cstdio>
 #include <cstdlib>
 #include <fstream>
 #include <sstream>
 #include <map>
+#include <string>
 
 PyPersistenceTest::PyPersistenceTest(
     const std::string& directory, const std::string& name)
@@ -103,7 +106,8 @@ void PyPersistenceTest::runTest()
 }
 
 //! Returns true if test output and reference file agree.
-bool PyPersistenceTest::compareFilePair(const std::string& dat_fpath, const std::string& ref_fpath)
+bool PyPersistenceTest::compareFilePair(
+    const std::string& dat_fpath, const std::string& ref_fpath) const
 {
     std::cout << "Comparing dat='" << dat_fpath << "' with ref='" << ref_fpath << "':\n";
     const std::string extension = Utils::String::split(FileSystem::filename(dat_fpath), ".")[2];
@@ -117,7 +121,7 @@ bool PyPersistenceTest::compareFilePair(const std::string& dat_fpath, const std:
 
 //! Returns true if intensity maps from test output and reference file agree reasonably.
 bool PyPersistenceTest::compareIntensityPair(
-    const std::string& dat_fpath, const std::string& ref_fpath)
+    const std::string& dat_fpath, const std::string& ref_fpath) const
 {
     const OutputData<double>* dat = IntensityDataIOFactory::readOutputData( dat_fpath );
     const OutputData<double>* ref = IntensityDataIOFactory::readOutputData( ref_fpath );
@@ -125,34 +129,76 @@ bool PyPersistenceTest::compareIntensityPair(
 }
 
 //! Returns true if YAML files from test output and reference agree.
-bool PyPersistenceTest::compareYamlPair(const std::string& dat_fpath, const std::string& ref_fpath)
+bool PyPersistenceTest::compareYamlPair(
+    const std::string& dat_fpath, const std::string& ref_fpath) const
 {
     std::fstream fdat(dat_fpath);
     std::fstream fref(ref_fpath);
-    const YAML::Node& dat = YAML::Load(fdat);
-    const YAML::Node& ref = YAML::Load(fref);
     return compareYamlNode( YAML::Load(fdat), YAML::Load(fref) );
 }
 
-    for (size_t i = 1; ; ++i) {
-        std::string datline;
-        std::string refline;
-        std::getline(fdat, datline);
-        std::getline(fref, refline);
-        if (datline!=refline) {
-            std::cerr << "Line " << i << " of " << dat_fpath << " and " << ref_fpath
-                      << " differs:\n";
-            std::cerr << "dat: '" << datline << "'\n";
-            std::cerr << "ref: '" << refline << "'\n";
-            return false;
+//! Returns true if all entries of the two YAML files agree.
+//! Floating-point entries must agree within a certain tolerance.
+//! The investigation is performed recursively, iterating over all lists and maps.
+bool PyPersistenceTest::compareYamlNode(const YAML::Node& dat, const YAML::Node& ref) const
+{
+//    std::cout << "DEBUG: " << dat.as<std::string>() << " vs " << ref.as<std::string>() << "\n";
+    if (dat.Type() != ref.Type()) {
+        std::cerr << "YAML node type differs: " << dat.Type() << " vs " << ref.Type() << "\n";
+        return false;
+    }
+    if      (dat.Type()==YAML::NodeType::Undefined)
+        throw std::runtime_error("Invalid node type 'Undefined' in YAML tree");
+    else if (dat.Type()==YAML::NodeType::Null)
+        throw std::runtime_error("Invalid node type 'Null' in YAML tree");
+    else if (dat.Type()==YAML::NodeType::Sequence) {
+        auto it_dat=dat.begin();
+        auto it_ref=ref.begin();
+        for (size_t idx=0; ; ++it_dat, ++it_ref, ++idx) {
+            if (it_dat==dat.end() || it_ref==ref.end()) {
+                if (!(it_dat==dat.end() && it_ref==ref.end())) {
+                    std::cerr << "lists have different length\n";
+                    return false;
+                }
+                return true; // regular exit
+            }
+            // now recurse into the node
+            if (!compareYamlNode(*it_dat, *it_ref)) {
+                std::cerr << "at list position " << idx << "\n";
+                return false;
+            }
         }
-        if (fdat.eof() && fref.eof())
-            break;
-        if (fdat.eof() || fref.eof()) {
-            std::cerr << "File length of " << dat_fpath << " and " << ref_fpath << " differs.\n";
+    } else if (dat.Type()==YAML::NodeType::Map) {
+        auto it_dat=dat.begin();
+        auto it_ref=ref.begin();
+        for ( ; ; ++it_dat, ++it_ref) {
+            if (it_dat==dat.end() || it_ref==ref.end()) {
+                if (!(it_dat==dat.end() && it_ref==ref.end())) {
+                    std::cerr << "maps have different length\n";
+                    return false;
+                }
+                return true; // regular exit
+            }
+            // now recurse into the node
+            if (!compareYamlNode(it_dat->second, it_ref->second)) {
+                std::cerr << "at map entry [" << it_dat->first << "]\n";
+                return false;
+            }
+        }
+    } else if (dat.Type()==YAML::NodeType::Scalar) {
+        if (dat.as<std::string>() == ref.as<std::string>())
+            return true;
+        try {
+            if (!Numeric::areAlmostEqual( dat.as<double>(), ref.as<double>(), 1e-10 )) {
+                std::cerr << "numbers differ: " << dat << " vs " << ref << "\n";
+                return false;
+            }
+            return true;
+        } catch(...) {
+            std::cerr << "scalar entries differ: '" << dat << "' vs '" << ref << "'\n";
             return false;
         }
-    }
-    std::cout << "Files fully agree.\n";
+    } else
+        throw std::runtime_error("Bug in PyPersistenceTest::compareYamlNode: unexpected node type");
     return true;
 }

Tests/Functional/PyCore/persistence/PyPersistenceTest.h

@@ -19,6 +19,10 @@
 #include "IReferencedTest.h"
 #include "OutputData.h"
 
+namespace YAML {
+    class Node;
+}
+
 //! @class PyPersistenceTest
 //! @ingroup standard_samples
 //! @brief A functional test that runs a Python script and compares its output with a reference.
@@ -32,12 +36,17 @@ public:
 
     void runTest() final;
 
-    bool compareFilePair(const std::string& dat_fname, const std::string& ref_fname);
-    bool compareIntensityPair(const std::string& dat_fname, const std::string& ref_fname);
-    bool compareYamlPair(const std::string& dat_fname, const std::string& ref_fname);
-
 private:
     std::string m_directory;
+
+    bool compareFilePair(
+        const std::string& dat_fname, const std::string& ref_fname) const;
+    bool compareIntensityPair(
+        const std::string& dat_fname, const std::string& ref_fname) const;
+    bool compareYamlPair(
+        const std::string& dat_fname, const std::string& ref_fname) const;
+    bool compareYamlNode(
+        const YAML::Node& dat, const YAML::Node& ref) const;
 };
 
 #endif // PYPERSISTENCETEST_H

Tests/Functional/TestMachinery/IReferencedTest.cpp

@@ -18,7 +18,7 @@
 #include <iostream>
 
 IFunctionalTest::ETestResult IReferencedTest::compareIntensityMaps(
-    const OutputData<double>& dat, const OutputData<double>& ref)
+    const OutputData<double>& dat, const OutputData<double>& ref) const
 {
     if( dat.getVariability() != ref.getVariability() ) {
         std::cerr << "Failed: reproducibility threshold has changed from "

Tests/Functional/TestMachinery/IReferencedTest.h

@@ -37,7 +37,8 @@ public:
 protected:
     double m_threshold;
 
-    ETestResult compareIntensityMaps(const OutputData<double>& dat, const OutputData<double>& ref);
+    ETestResult compareIntensityMaps(
+        const OutputData<double>& dat, const OutputData<double>& ref) const;
 };
 
 #endif // IREFERENCEDTEST_H