diff --git a/Applications/ApplicationsLib/TestDefinition.cpp b/Applications/ApplicationsLib/TestDefinition.cpp
index 5e0aa14bfbce239f24f71fb3ae0294e70dcfbbba..4641fadb68e68c430a08f7302983eec65f247856 100644
--- a/Applications/ApplicationsLib/TestDefinition.cpp
+++ b/Applications/ApplicationsLib/TestDefinition.cpp
@@ -204,6 +204,15 @@ TestDefinition::TestDefinition(BaseLib::ConfigTree const& config_tree,
             filenames.push_back(filename);
         }
 
+        if (empty(filenames))
+        {
+            OGS_FATAL(
+                "No files from test definitions were added for tests but {} "
+                "{:s} specified.",
+                size(vtkdiff_configs),
+                (size(vtkdiff_configs) == 1 ? "test was" : "tests were"));
+        }
+
         auto const absolute_tolerance =
             //! \ogs_file_param{prj__test_definition__vtkdiff__absolute_tolerance}
             vtkdiff_config.getConfigParameter<std::string>("absolute_tolerance",
@@ -278,4 +287,9 @@ std::vector<std::string> const& TestDefinition::getOutputFiles() const
 {
     return _output_files;
 }
+
+std::size_t TestDefinition::numberOfTests() const
+{
+    return size(_command_lines);
+}
 }  // namespace ApplicationsLib
diff --git a/Applications/ApplicationsLib/TestDefinition.h b/Applications/ApplicationsLib/TestDefinition.h
index a562544e19247e8714a70693a9f23db92776f5b2..668f7a9bf3964c0034cba17b95aa8138a7657f53 100644
--- a/Applications/ApplicationsLib/TestDefinition.h
+++ b/Applications/ApplicationsLib/TestDefinition.h
@@ -34,6 +34,7 @@ public:
 
     bool runTests() const;
     std::vector<std::string> const& getOutputFiles() const;
+    std::size_t numberOfTests() const;
 
 private:
     std::vector<std::string> _command_lines;
diff --git a/Applications/CLI/ogs.cpp b/Applications/CLI/ogs.cpp
index 5d8fff6658a191ebdee644512fc7efe210204979..1f499d35371a11d57c1a8ea8f01cf40e30991844 100644
--- a/Applications/CLI/ogs.cpp
+++ b/Applications/CLI/ogs.cpp
@@ -68,7 +68,8 @@ int main(int argc, char* argv[])
     TCLAP::ValueArg<std::string> reference_path_arg(
         "r", "reference",
         "Run output result comparison after successful simulation comparing to "
-        "all files in the given path.",
+        "all files in the given path. This requires test definitions to be "
+        "present in the project file.",
         false, "", "PATH");
     cmd.add(reference_path_arg);
 
@@ -204,6 +205,12 @@ int main(int argc, char* argv[])
                         project_config->getConfigSubtree("test_definition"),
                         reference_path_arg.getValue(),
                         outdir_arg.getValue());
+                if( test_definition->numberOfTests() == 0)
+                {
+                    OGS_FATAL(
+                        "No tests were constructed from the test definitions, "
+                        "but reference solutions path was given.");
+                }
 
                 INFO("Cleanup possible output files before running ogs.");
                 BaseLib::removeFiles(test_definition->getOutputFiles());