Benchmark.cpp
1 /*********************************************************************
2 * Software License Agreement (BSD License)
3 *
4 * Copyright (c) 2010, Rice University
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 * * Neither the name of the Rice University nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 *********************************************************************/
34 
35 /* Author: Ioan Sucan, Luis G. Torres */
36 
37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include "ompl/config.h"
41 #include <boost/scoped_ptr.hpp>
42 #include <boost/progress.hpp>
43 #include <thread>
44 #include <mutex>
45 #include <condition_variable>
46 #include <fstream>
47 #include <sstream>
48 
50 namespace ompl
51 {
52  namespace tools
53  {
56  static std::string getResultsFilename(const Benchmark::CompleteExperiment &exp)
57  {
58  return "ompl_" + exp.host + "_" + time::as_string(exp.startTime) + ".log";
59  }
60 
63  static std::string getConsoleFilename(const Benchmark::CompleteExperiment &exp)
64  {
65  return "ompl_" + exp.host + "_" + time::as_string(exp.startTime) + ".console";
66  }
67 
68  static bool terminationCondition(const machine::MemUsage_t maxMem, const time::point &endTime)
69  {
70  if (time::now() < endTime && machine::getProcessMemoryUsage() < maxMem)
71  return false;
72  return true;
73  }
74 
75  class RunPlanner
76  {
77  public:
78  RunPlanner(const Benchmark *benchmark, bool useThreads)
79  : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
80  {
81  }
82 
83  void run(const base::PlannerPtr &planner, const machine::MemUsage_t memStart,
84  const machine::MemUsage_t maxMem, const double maxTime, const double timeBetweenUpdates)
85  {
86  // if (!useThreads_)
87  // {
88  runThread(planner, memStart + maxMem, time::seconds(maxTime), time::seconds(timeBetweenUpdates));
89  // return;
90  // }
91 
92  // std::thread t([planner, memStart, maxMem, maxTime, timeBetweenUpdates]
93  // {
94  // runThread(planner, memStart + maxMem, time::seconds(maxTime),
95  // time::seconds(timeBetweenUpdates)); });
96  // }
97 
98  // allow 25% more time than originally specified, in order to detect planner termination
99  // if (!t.try_join_for(time::seconds(maxTime * 1.25)))
100  // {
101  // status_ = base::PlannerStatus::CRASH;
102  //
103  // std::stringstream es;
104  // es << "Planner " << benchmark_->getStatus().activePlanner << " did not complete run " <<
105  // benchmark_->getStatus().activeRun
106  // << " within the specified amount of time (possible crash). Attempting to force termination of
107  // planning thread ..." << std::endl;
108  // std::cerr << es.str();
109  // OMPL_ERROR(es.str().c_str());
110  //
111  // t.interrupt();
112  // t.join();
113  //
114  // std::string m = "Planning thread cancelled";
115  // std::cerr << m << std::endl;
116  // OMPL_ERROR(m.c_str());
117  // }
118 
119  // if (memStart < memUsed_)
120  // memUsed_ -= memStart;
121  // else
122  // memUsed_ = 0;
123  }
124 
125  double getTimeUsed() const
126  {
127  return timeUsed_;
128  }
129 
130  machine::MemUsage_t getMemUsed() const
131  {
132  return memUsed_;
133  }
134 
135  base::PlannerStatus getStatus() const
136  {
137  return status_;
138  }
139 
140  const Benchmark::RunProgressData &getRunProgressData() const
141  {
142  return runProgressData_;
143  }
144 
145  private:
146  void runThread(const base::PlannerPtr &planner, const machine::MemUsage_t maxMem,
147  const time::duration &maxDuration, const time::duration &timeBetweenUpdates)
148  {
149  time::point timeStart = time::now();
150 
151  try
152  {
153  const time::point endtime = time::now() + maxDuration;
154  base::PlannerTerminationConditionFn ptc([maxMem, endtime]
155  {
156  return terminationCondition(maxMem, endtime);
157  });
158  solved_ = false;
159  // Only launch the planner progress property
160  // collector if there is any data for it to report
161  //
162  // \todo issue here is that at least one sample
163  // always gets taken before planner even starts;
164  // might be worth adding a short wait time before
165  // collector begins sampling
166  boost::scoped_ptr<std::thread> t;
167  if (planner->getPlannerProgressProperties().size() > 0)
168  t.reset(new std::thread([this, &planner, timeBetweenUpdates]
169  {
170  collectProgressProperties(planner->getPlannerProgressProperties(),
171  timeBetweenUpdates);
172  }));
173  status_ = planner->solve(ptc, 0.1);
174  solvedFlag_.lock();
175  solved_ = true;
176  solvedCondition_.notify_all();
177  solvedFlag_.unlock();
178  if (t)
179  t->join(); // maybe look into interrupting even if planner throws an exception
180  }
181  catch (std::runtime_error &e)
182  {
183  std::stringstream es;
184  es << "There was an error executing planner " << benchmark_->getStatus().activePlanner
185  << ", run = " << benchmark_->getStatus().activeRun << std::endl;
186  es << "*** " << e.what() << std::endl;
187  std::cerr << es.str();
188  OMPL_ERROR(es.str().c_str());
189  }
190 
191  timeUsed_ = time::seconds(time::now() - timeStart);
192  memUsed_ = machine::getProcessMemoryUsage();
193  }
194 
195  void collectProgressProperties(const base::Planner::PlannerProgressProperties &properties,
196  const time::duration &timePerUpdate)
197  {
198  time::point timeStart = time::now();
199 
200  std::unique_lock<std::mutex> ulock(solvedFlag_);
201  while (!solved_)
202  {
203  if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
204  return;
205  else
206  {
207  double timeInSeconds = time::seconds(time::now() - timeStart);
208  std::string timeStamp = std::to_string(timeInSeconds);
209  std::map<std::string, std::string> data;
210  data["time REAL"] = timeStamp;
211  for (const auto &property : properties)
212  {
213  data[property.first] = property.second();
214  }
215  runProgressData_.push_back(data);
216  }
217  }
218  }
219 
220  const Benchmark *benchmark_;
221  double timeUsed_;
222  machine::MemUsage_t memUsed_;
223  base::PlannerStatus status_;
224  bool useThreads_;
225  Benchmark::RunProgressData runProgressData_;
226 
227  // variables needed for progress property collection
228  bool solved_;
229  std::mutex solvedFlag_;
230  std::condition_variable solvedCondition_;
231  };
232  }
233 }
235 
236 bool ompl::tools::Benchmark::saveResultsToFile(const char *filename) const
237 {
238  bool result = false;
239 
240  std::ofstream fout(filename);
241  if (fout.good())
242  {
243  result = saveResultsToStream(fout);
244  OMPL_INFORM("Results saved to '%s'", filename);
245  }
246  else
247  {
248  // try to save to a different file, if we can
249  if (getResultsFilename(exp_) != std::string(filename))
250  result = saveResultsToFile();
251 
252  OMPL_ERROR("Unable to write results to '%s'", filename);
253  }
254  return result;
255 }
256 
258 {
259  std::string filename = getResultsFilename(exp_);
260  return saveResultsToFile(filename.c_str());
261 }
262 
263 bool ompl::tools::Benchmark::saveResultsToStream(std::ostream &out) const
264 {
265  if (exp_.planners.empty())
266  {
267  OMPL_WARN("There is no experimental data to save");
268  return false;
269  }
270 
271  if (!out.good())
272  {
273  OMPL_ERROR("Unable to write to stream");
274  return false;
275  }
276 
277  out << "OMPL version " << OMPL_VERSION << std::endl;
278  out << "Experiment " << (exp_.name.empty() ? "NO_NAME" : exp_.name) << std::endl;
279 
280  out << exp_.parameters.size() << " experiment properties" << std::endl;
281  for (const auto &parameter : exp_.parameters)
282  out << parameter.first << " = " << parameter.second << std::endl;
283 
284  out << "Running on " << (exp_.host.empty() ? "UNKNOWN" : exp_.host) << std::endl;
285  out << "Starting at " << time::as_string(exp_.startTime) << std::endl;
286  out << "<<<|" << std::endl
287  << exp_.setupInfo << "|>>>" << std::endl;
288  out << "<<<|" << std::endl
289  << exp_.cpuInfo << "|>>>" << std::endl;
290 
291  out << exp_.seed << " is the random seed" << std::endl;
292  out << exp_.maxTime << " seconds per run" << std::endl;
293  out << exp_.maxMem << " MB per run" << std::endl;
294  out << exp_.runCount << " runs per planner" << std::endl;
295  out << exp_.totalDuration << " seconds spent to collect the data" << std::endl;
296 
297  // change this if more enum types are added
298  out << "1 enum type" << std::endl;
299  out << "status";
300  for (unsigned int i = 0; i < base::PlannerStatus::TYPE_COUNT; ++i)
301  out << '|' << base::PlannerStatus(static_cast<base::PlannerStatus::StatusType>(i)).asString();
302  out << std::endl;
303 
304  out << exp_.planners.size() << " planners" << std::endl;
305 
306  for (const auto &planner : exp_.planners)
307  {
308  out << planner.name << std::endl;
309 
310  // get names of common properties
311  std::vector<std::string> properties;
312  for (auto &property : planner.common)
313  properties.push_back(property.first);
314  std::sort(properties.begin(), properties.end());
315 
316  // print names & values of common properties
317  out << properties.size() << " common properties" << std::endl;
318  for (auto &property : properties)
319  {
320  auto it = planner.common.find(property);
321  out << it->first << " = " << it->second << std::endl;
322  }
323 
324  // construct the list of all possible properties for all runs
325  std::map<std::string, bool> propSeen;
326  for (auto &run : planner.runs)
327  for (auto &property : run)
328  propSeen[property.first] = true;
329 
330  properties.clear();
331 
332  for (auto &it : propSeen)
333  properties.push_back(it.first);
334  std::sort(properties.begin(), properties.end());
335 
336  // print the property names
337  out << properties.size() << " properties for each run" << std::endl;
338  for (auto &propertie : properties)
339  out << propertie << std::endl;
340 
341  // print the data for each run
342  out << planner.runs.size() << " runs" << std::endl;
343  for (auto &run : planner.runs)
344  {
345  for (auto &property : properties)
346  {
347  auto it = run.find(property);
348  if (it != run.end())
349  out << it->second;
350  out << "; ";
351  }
352  out << std::endl;
353  }
354 
355  // print the run progress data if it was reported
356  if (planner.runsProgressData.size() > 0)
357  {
358  // Print number of progress properties
359  out << planner.progressPropertyNames.size() << " progress properties for each run" << std::endl;
360  // Print progress property names
361  for (const auto &progPropName : planner.progressPropertyNames)
362  {
363  out << progPropName << std::endl;
364  }
365  // Print progress properties for each run
366  out << planner.runsProgressData.size() << " runs" << std::endl;
367  for (const auto &r : planner.runsProgressData)
368  {
369  // For each time point
370  for (const auto &t : r)
371  {
372  // Print each of the properties at that time point
373  for (const auto &iter : t)
374  {
375  out << iter.second << ",";
376  }
377 
378  // Separate time points by semicolons
379  out << ";";
380  }
381 
382  // Separate runs by newlines
383  out << std::endl;
384  }
385  }
386 
387  out << '.' << std::endl;
388  }
389  return true;
390 }
391 
393 {
394  // sanity checks
395  if (gsetup_)
396  {
397  if (!gsetup_->getSpaceInformation()->isSetup())
398  gsetup_->getSpaceInformation()->setup();
399  }
400  else
401  {
402  if (!csetup_->getSpaceInformation()->isSetup())
403  csetup_->getSpaceInformation()->setup();
404  }
405 
406  if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
407  {
408  OMPL_ERROR("No goal defined");
409  return;
410  }
411 
412  if (planners_.empty())
413  {
414  OMPL_ERROR("There are no planners to benchmark");
415  return;
416  }
417 
418  status_.running = true;
419  exp_.totalDuration = 0.0;
420  exp_.maxTime = req.maxTime;
421  exp_.maxMem = req.maxMem;
422  exp_.runCount = req.runCount;
423  exp_.host = machine::getHostname();
424  exp_.cpuInfo = machine::getCPUInfo();
425  exp_.seed = RNG::getSeed();
426 
427  exp_.startTime = time::now();
428 
429  OMPL_INFORM("Configuring planners ...");
430 
431  // clear previous experimental data
432  exp_.planners.clear();
433  exp_.planners.resize(planners_.size());
434 
435  const base::ProblemDefinitionPtr &pdef =
436  gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
437  // set up all the planners
438  for (unsigned int i = 0; i < planners_.size(); ++i)
439  {
440  // configure the planner
441  planners_[i]->setProblemDefinition(pdef);
442  if (!planners_[i]->isSetup())
443  planners_[i]->setup();
444  exp_.planners[i].name = (gsetup_ ? "geometric_" : "control_") + planners_[i]->getName();
445  OMPL_INFORM("Configured %s", exp_.planners[i].name.c_str());
446  }
447 
448  OMPL_INFORM("Done configuring planners.");
449  OMPL_INFORM("Saving planner setup information ...");
450 
451  std::stringstream setupInfo;
452  if (gsetup_)
453  gsetup_->print(setupInfo);
454  else
455  csetup_->print(setupInfo);
456  setupInfo << std::endl
457  << "Properties of benchmarked planners:" << std::endl;
458  for (auto &planner : planners_)
459  planner->printProperties(setupInfo);
460 
461  exp_.setupInfo = setupInfo.str();
462 
463  OMPL_INFORM("Done saving information");
464 
465  OMPL_INFORM("Beginning benchmark");
467  boost::scoped_ptr<msg::OutputHandlerFile> ohf;
468  if (req.saveConsoleOutput)
469  {
470  ohf.reset(new msg::OutputHandlerFile(getConsoleFilename(exp_).c_str()));
471  msg::useOutputHandler(ohf.get());
472  }
473  else
475  OMPL_INFORM("Beginning benchmark");
476 
477  boost::scoped_ptr<boost::progress_display> progress;
478  if (req.displayProgress)
479  {
480  std::cout << "Running experiment " << exp_.name << "." << std::endl;
481  std::cout << "Each planner will be executed " << req.runCount << " times for at most " << req.maxTime
482  << " seconds. Memory is limited at " << req.maxMem << "MB." << std::endl;
483  progress.reset(new boost::progress_display(100, std::cout));
484  }
485 
487  auto maxMemBytes = (machine::MemUsage_t)(req.maxMem * 1024 * 1024);
488 
489  for (unsigned int i = 0; i < planners_.size(); ++i)
490  {
491  status_.activePlanner = exp_.planners[i].name;
492  // execute planner switch event, if set
493  try
494  {
495  if (plannerSwitch_)
496  {
497  OMPL_INFORM("Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
498  plannerSwitch_(planners_[i]);
499  OMPL_INFORM("Completed execution of planner-switch event");
500  }
501  }
502  catch (std::runtime_error &e)
503  {
504  std::stringstream es;
505  es << "There was an error executing the planner-switch event for planner " << status_.activePlanner
506  << std::endl;
507  es << "*** " << e.what() << std::endl;
508  std::cerr << es.str();
509  OMPL_ERROR(es.str().c_str());
510  }
511  if (gsetup_)
512  gsetup_->setup();
513  else
514  csetup_->setup();
515  planners_[i]->params().getParams(exp_.planners[i].common);
516  planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
517 
518  // Add planner progress property names to struct
519  exp_.planners[i].progressPropertyNames.emplace_back("time REAL");
520  base::Planner::PlannerProgressProperties::const_iterator iter;
521  for (iter = planners_[i]->getPlannerProgressProperties().begin();
522  iter != planners_[i]->getPlannerProgressProperties().end(); ++iter)
523  {
524  exp_.planners[i].progressPropertyNames.push_back(iter->first);
525  }
526  std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
527 
528  // run the planner
529  for (unsigned int j = 0; j < req.runCount; ++j)
530  {
531  status_.activeRun = j;
532  status_.progressPercentage =
533  (double)(100 * (req.runCount * i + j)) / (double)(planners_.size() * req.runCount);
534 
535  if (req.displayProgress)
536  while (status_.progressPercentage > progress->count())
537  ++(*progress);
538 
539  OMPL_INFORM("Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
540 
541  // make sure all planning data structures are cleared
542  try
543  {
544  planners_[i]->clear();
545  if (gsetup_)
546  {
547  gsetup_->getProblemDefinition()->clearSolutionPaths();
548  gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
549  }
550  else
551  {
552  csetup_->getProblemDefinition()->clearSolutionPaths();
553  csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
554  }
555  }
556  catch (std::runtime_error &e)
557  {
558  std::stringstream es;
559  es << "There was an error while preparing for run " << status_.activeRun << " of planner "
560  << status_.activePlanner << std::endl;
561  es << "*** " << e.what() << std::endl;
562  std::cerr << es.str();
563  OMPL_ERROR(es.str().c_str());
564  }
565 
566  // execute pre-run event, if set
567  try
568  {
569  if (preRun_)
570  {
571  OMPL_INFORM("Executing pre-run event for run %d of planner %s ...", status_.activeRun,
572  status_.activePlanner.c_str());
573  preRun_(planners_[i]);
574  OMPL_INFORM("Completed execution of pre-run event");
575  }
576  }
577  catch (std::runtime_error &e)
578  {
579  std::stringstream es;
580  es << "There was an error executing the pre-run event for run " << status_.activeRun << " of planner "
581  << status_.activePlanner << std::endl;
582  es << "*** " << e.what() << std::endl;
583  std::cerr << es.str();
584  OMPL_ERROR(es.str().c_str());
585  }
586 
587  RunPlanner rp(this, req.useThreads);
588  rp.run(planners_[i], memStart, maxMemBytes, req.maxTime, req.timeBetweenUpdates);
589  bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
590 
591  // store results
592  try
593  {
594  RunProperties run;
595 
596  run["time REAL"] = std::to_string(rp.getTimeUsed());
597  run["memory REAL"] = std::to_string((double)rp.getMemUsed() / (1024.0 * 1024.0));
598  run["status ENUM"] = std::to_string((int)static_cast<base::PlannerStatus::StatusType>(rp.getStatus()));
599  if (gsetup_)
600  {
601  run["solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
602  run["valid segment fraction REAL"] =
603  std::to_string(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
604  }
605  else
606  {
607  run["solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
608  run["valid segment fraction REAL"] =
609  std::to_string(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
610  }
611 
612  if (solved)
613  {
614  if (gsetup_)
615  {
616  run["approximate solution BOOLEAN"] =
617  std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
618  run["solution difference REAL"] =
619  std::to_string(gsetup_->getProblemDefinition()->getSolutionDifference());
620  run["solution length REAL"] = std::to_string(gsetup_->getSolutionPath().length());
621  run["solution smoothness REAL"] = std::to_string(gsetup_->getSolutionPath().smoothness());
622  run["solution clearance REAL"] = std::to_string(gsetup_->getSolutionPath().clearance());
623  run["solution segments INTEGER"] =
624  std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
625  run["correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
626 
627  unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
628  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
629  run["correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
630  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
631 
632  if (req.simplify)
633  {
634  // simplify solution
635  time::point timeStart = time::now();
636  gsetup_->simplifySolution();
637  double timeUsed = time::seconds(time::now() - timeStart);
638  run["simplification time REAL"] = std::to_string(timeUsed);
639  run["simplified solution length REAL"] =
640  std::to_string(gsetup_->getSolutionPath().length());
641  run["simplified solution smoothness REAL"] =
642  std::to_string(gsetup_->getSolutionPath().smoothness());
643  run["simplified solution clearance REAL"] =
644  std::to_string(gsetup_->getSolutionPath().clearance());
645  run["simplified solution segments INTEGER"] =
646  std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
647  run["simplified correct solution BOOLEAN"] =
648  std::to_string(gsetup_->getSolutionPath().check());
649  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
650  run["simplified correct solution strict BOOLEAN"] =
651  std::to_string(gsetup_->getSolutionPath().check());
652  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
653  }
654  }
655  else
656  {
657  run["approximate solution BOOLEAN"] =
658  std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
659  run["solution difference REAL"] =
660  std::to_string(csetup_->getProblemDefinition()->getSolutionDifference());
661  run["solution length REAL"] = std::to_string(csetup_->getSolutionPath().length());
662  run["solution clearance REAL"] =
663  std::to_string(csetup_->getSolutionPath().asGeometric().clearance());
664  run["solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
665  run["correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
666  }
667  }
668 
669  base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
670  planners_[i]->getPlannerData(pd);
671  run["graph states INTEGER"] = std::to_string(pd.numVertices());
672  run["graph motions INTEGER"] = std::to_string(pd.numEdges());
673 
674  for (std::map<std::string, std::string>::const_iterator it = pd.properties.begin();
675  it != pd.properties.end(); ++it)
676  run[it->first] = it->second;
677 
678  // execute post-run event, if set
679  try
680  {
681  if (postRun_)
682  {
683  OMPL_INFORM("Executing post-run event for run %d of planner %s ...", status_.activeRun,
684  status_.activePlanner.c_str());
685  postRun_(planners_[i], run);
686  OMPL_INFORM("Completed execution of post-run event");
687  }
688  }
689  catch (std::runtime_error &e)
690  {
691  std::stringstream es;
692  es << "There was an error in the execution of the post-run event for run " << status_.activeRun
693  << " of planner " << status_.activePlanner << std::endl;
694  es << "*** " << e.what() << std::endl;
695  std::cerr << es.str();
696  OMPL_ERROR(es.str().c_str());
697  }
698 
699  exp_.planners[i].runs.push_back(run);
700 
701  // Add planner progress data from the planner progress
702  // collector if there was anything to report
703  if (planners_[i]->getPlannerProgressProperties().size() > 0)
704  {
705  exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
706  }
707  }
708  catch (std::runtime_error &e)
709  {
710  std::stringstream es;
711  es << "There was an error in the extraction of planner results: planner = " << status_.activePlanner
712  << ", run = " << status_.activePlanner << std::endl;
713  es << "*** " << e.what() << std::endl;
714  std::cerr << es.str();
715  OMPL_ERROR(es.str().c_str());
716  }
717  }
718  }
719 
720  status_.running = false;
721  status_.progressPercentage = 100.0;
722  if (req.displayProgress)
723  {
724  while (status_.progressPercentage > progress->count())
725  ++(*progress);
726  std::cout << std::endl;
727  }
728 
729  exp_.totalDuration = time::seconds(time::now() - exp_.startTime);
730 
731  OMPL_INFORM("Benchmark complete");
733  OMPL_INFORM("Benchmark complete");
734 }
std::string getHostname()
Get the hostname of the machine in use.
std::map< std::string, PlannerProgressProperty > PlannerProgressProperties
A dictionary which maps the name of a progress property to the function to be used for querying that ...
Definition: Planner.h:413
MemUsage_t getProcessMemoryUsage()
Get the amount of memory the current process is using. This should work on major platforms (Windows,...
void noOutputHandler()
This function instructs ompl that no messages should be outputted. Equivalent to useOutputHandler(nul...
Definition: Console.cpp:95
Generic class to handle output from a piece of code.
Definition: Console.h:102
void useOutputHandler(OutputHandler *oh)
Specify the instance of the OutputHandler to use. By default, this is OutputHandlerSTD.
Definition: Console.cpp:108
CompleteExperiment exp_
The collected experimental data (for all planners)
Definition: Benchmark.h:439
std::string asString() const
Return a string representation.
@ TYPE_COUNT
The number of possible status values.
point now()
Get the current time point.
Definition: Time.h:134
std::chrono::system_clock::time_point point
Representation of a point in time.
Definition: Time.h:128
unsigned int numEdges() const
Retrieve the number of edges in this structure.
#define OMPL_INFORM(fmt,...)
Log a formatted information string.
Definition: Console.h:68
bool simplify
flag indicating whether simplification should be applied to path; true by default
Definition: Benchmark.h:295
bool useThreads
flag indicating whether planner runs should be run in a separate thread. It is advisable to set this ...
Definition: Benchmark.h:292
Object containing planner generated vertex and edge data. It is assumed that all vertices are unique,...
Definition: PlannerData.h:238
unsigned int runCount
the number of times to run each planner; 100 by default
Definition: Benchmark.h:276
bool saveConsoleOutput
flag indicating whether console output is saved (in an automatically generated filename); true by def...
Definition: Benchmark.h:287
unsigned int numVertices() const
Retrieve the number of vertices in this structure.
bool displayProgress
flag indicating whether progress is to be displayed or not; true by default
Definition: Benchmark.h:283
bool saveResultsToFile() const
Save the results of the benchmark to a file. The name of the file is the current date and time.
Definition: Benchmark.cpp:257
A class to store the exit status of Planner::solve()
Representation of a benchmark request.
Definition: Benchmark.h:252
std::map< std::string, std::string > RunProperties
The data collected from a run of a planner is stored as key-value pairs.
Definition: Benchmark.h:175
Implementation of OutputHandler that saves messages in a file.
Definition: Console.h:125
static std::uint_fast32_t getSeed()
Get the seed used to generate the seeds of each RNG instance. Passing the returned value to setSeed()...
#define OMPL_WARN(fmt,...)
Log a formatted warning string.
Definition: Console.h:66
std::map< std::string, std::string > properties
Any extra properties (key-value pairs) the planner can set.
Definition: PlannerData.h:470
virtual void benchmark(const Request &req)
Benchmark the added planners on the defined problem. Repeated calls clear previously gathered data.
Definition: Benchmark.cpp:392
std::string getCPUInfo()
Get information about the CPU of the machine in use.
StatusType
The possible values of the status returned by a planner.
double maxMem
the maximum amount of memory a planner is allowed to use (MB); 4096.0 by default
Definition: Benchmark.h:273
std::function< bool()> PlannerTerminationConditionFn
Signature for functions that decide whether termination conditions have been met for a planner,...
#define OMPL_ERROR(fmt,...)
Log a formatted error string.
Definition: Console.h:64
unsigned long long MemUsage_t
Amount of memory used, in bytes.
Definition: MachineSpecs.h:112
std::string as_string(const point &p)
Return string representation of point in time.
Definition: Time.h:154
OutputHandler * getOutputHandler()
Get the instance of the OutputHandler currently used. This is nullptr in case there is no output hand...
Definition: Console.cpp:115
duration seconds(double sec)
Return the time duration representing a given number of seconds.
Definition: Time.h:140
virtual bool saveResultsToStream(std::ostream &out=std::cout) const
Save the results of the benchmark to a stream.
Definition: Benchmark.cpp:263
double timeBetweenUpdates
When collecting time-varying data from a planner during its execution, the planner's progress will be...
Definition: Benchmark.h:280
Main namespace. Contains everything in this library.
Definition: Cost.h:42
double maxTime
the maximum amount of time a planner is allowed to run (seconds); 5.0 by default
Definition: Benchmark.h:270
std::chrono::system_clock::duration duration
Representation of a time duration.
Definition: Time.h:131