178 typedef typename Model::ModelParameters ModelParameters;
179 typedef typename Solver::SolverParameters SolverParameters;
205 : ebosSimulator_(ebosSimulator)
210 this->terminalOutput_ =
false;
211 if (this->grid().comm().
rank() == 0) {
212 this->terminalOutput_ =
EWOMS_GET_PARAM(TypeTag,
bool, EnableTerminalOutput);
214 this->startConvergenceOutputThread(
EWOMS_GET_PARAM(TypeTag, std::string,
215 OutputExtraConvergenceInfo),
216 R
"(OutputExtraConvergenceInfo (--output-extra-convergence-info))");
236 const auto&
ioconfig = ebosSimulator_.vanguard().eclState().getIOConfig();
240 std::filesystem::path
path(
ioconfig.getInputDir() +
"/");
241 path.replace_filename(
ioconfig.getBaseName() +
".OPMRST");
243 if (!std::filesystem::exists(
loadFile_)) {
253 this->endConvergenceOutputThread();
256 static void registerParameters()
258 ModelParameters::registerParameters();
259 SolverParameters::registerParameters();
260 TimeStepper::registerParameters();
262 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableTerminalOutput,
263 "Print high-level information about the simulation's progress to the terminal");
264 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableAdaptiveTimeStepping,
265 "Use adaptive time stepping between report steps");
266 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableTuning,
267 "Honor some aspects of the TUNING keyword.");
268 EWOMS_REGISTER_PARAM(TypeTag, std::string, OutputExtraConvergenceInfo,
269 "Provide additional convergence output "
270 "files for diagnostic purposes. "
271 "\"none\" gives no extra output and "
272 "overrides all other options, "
273 "\"steps\" generates an INFOSTEP file, "
274 "\"iterations\" generates an INFOITER file. "
275 "Combine options with commas, e.g., "
276 "\"steps,iterations\" for multiple outputs.");
277 EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveStep,
278 "Save serialized state to .OPMRST file. "
279 "Either a specific report step, \"all\" to save "
280 "all report steps or \":x\" to save every x'th step."
281 "Use negative values of \"x\" to keep only the last "
282 "written step, or \"last\" to save every step, keeping "
284 EWOMS_REGISTER_PARAM(TypeTag,
int, LoadStep,
285 "Load serialized state from .OPMRST file. "
286 "Either a specific report step, or 0 to load last "
287 "stored report step.");
288 EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveFile,
289 "FileName for .OPMRST file used for saving serialized state. "
290 "If empty, CASENAME.OPMRST is used.");
291 EWOMS_HIDE_PARAM(TypeTag, SaveFile);
292 EWOMS_REGISTER_PARAM(TypeTag, std::string, LoadFile,
293 "FileName for .OPMRST file used to load serialized state. "
294 "If empty, CASENAME.OPMRST is used.");
295 EWOMS_HIDE_PARAM(TypeTag, LoadFile);
308 ebosSimulator_.model().invalidateAndUpdateIntensiveQuantities(0);
310 while (!
timer.done()) {
319 ebosSimulator_.setEpisodeIndex(-1);
322 solverTimer_ = std::make_unique<time::StopWatch>();
323 totalTimer_ = std::make_unique<time::StopWatch>();
324 totalTimer_->start();
334 adaptiveTimeStepping_ = std::make_unique<TimeStepper>(
max_next_tstep,
339 adaptiveTimeStepping_ = std::make_unique<TimeStepper>(unitSystem, terminalOutput_);
345 adaptiveTimeStepping_->setSuggestedNextStep(ebosSimulator_.timeStepSize());
350 void updateTUNING(
const Tuning& tuning) {
351 modelParam_.tolerance_mb_ = tuning.XXXMBE;
352 if (terminalOutput_) {
353 OpmLog::debug(fmt::format(
"Setting SimulatorFullyImplicitBlackoilEbos mass balance limit (XXXMBE) to {:.2e}", tuning.XXXMBE));
357 bool runStep(SimulatorTimer& timer)
359 if (schedule().exitStatus().has_value()) {
360 if (terminalOutput_) {
361 OpmLog::info(
"Stopping simulation since EXIT was triggered by an action keyword.");
363 report_.success.exit_status = schedule().exitStatus().value();
372 if (terminalOutput_) {
373 std::ostringstream ss;
375 OpmLog::debug(ss.str());
378 if (terminalOutput_) {
379 outputReportStep(timer);
383 if (timer.initialStep()) {
384 Dune::Timer perfTimer;
387 ebosSimulator_.setEpisodeIndex(-1);
388 ebosSimulator_.setEpisodeLength(0.0);
389 ebosSimulator_.setTimeStepSize(0.0);
390 wellModel_().beginReportStep(timer.currentStepNum());
391 ebosSimulator_.problem().writeOutput();
393 report_.success.output_write_time += perfTimer.stop();
397 solverTimer_->start();
400 solver_ = createSolver(wellModel_());
403 ebosSimulator_.startNextEpisode(
404 ebosSimulator_.startTime()
405 + schedule().seconds(timer.currentStepNum()),
406 timer.currentStepLength());
407 ebosSimulator_.setEpisodeIndex(timer.currentStepNum());
409 wellModel_().prepareDeserialize(
loadStep_ - 1);
412 ebosSimulator_.model().invalidateAndUpdateIntensiveQuantities(0);
414 solver_->model().beginReportStep();
415 bool enableTUNING = EWOMS_GET_PARAM(TypeTag,
bool, EnableTuning);
422 if (adaptiveTimeStepping_) {
423 const auto& events = schedule()[timer.currentStepNum()].events();
425 if (events.hasEvent(ScheduleEvents::TUNING_CHANGE)) {
426 const auto& sched_state = schedule()[timer.currentStepNum()];
427 const auto& tuning = sched_state.tuning();
428 const auto& max_next_tstep = sched_state.max_next_tstep();
429 adaptiveTimeStepping_->updateTUNING(max_next_tstep, tuning);
432 solver_->model().updateTUNING(tuning);
433 this->updateTUNING(tuning);
436 bool event = events.hasEvent(ScheduleEvents::NEW_WELL) ||
437 events.hasEvent(ScheduleEvents::INJECTION_TYPE_CHANGED) ||
438 events.hasEvent(ScheduleEvents::WELL_SWITCHED_INJECTOR_PRODUCER) ||
439 events.hasEvent(ScheduleEvents::PRODUCTION_UPDATE) ||
440 events.hasEvent(ScheduleEvents::INJECTION_UPDATE) ||
441 events.hasEvent(ScheduleEvents::WELL_STATUS_CHANGE);
442 auto stepReport = adaptiveTimeStepping_->step(timer, *solver_, event,
nullptr);
443 report_ += stepReport;
445 ebosSimulator_.problem().setSimulationReport(report_);
448 auto stepReport = solver_->step(timer);
449 report_ += stepReport;
450 if (terminalOutput_) {
451 std::ostringstream ss;
452 stepReport.reportStep(ss);
453 OpmLog::info(ss.str());
458 Dune::Timer perfTimer;
460 const double nextstep = adaptiveTimeStepping_ ? adaptiveTimeStepping_->suggestedNextStep() : -1.0;
461 ebosSimulator_.problem().setNextTimeStepSize(nextstep);
462 ebosSimulator_.problem().writeOutput();
463 report_.success.output_write_time += perfTimer.stop();
465 solver_->model().endReportStep();
468 solverTimer_->stop();
471 report_.success.solver_time += solverTimer_->secsSinceStart();
473 if (this->grid().comm().rank() == 0) {
475 const auto& reps = solver_->model().stepReports();
476 this->writeConvergenceOutput(std::vector<StepReport>{reps.begin() + already_reported_steps_, reps.end()});
477 already_reported_steps_ = reps.size();
483 if (terminalOutput_) {
484 if (!timer.initialStep()) {
486 outputTimestampFIP(timer, eclState().getTitle(), version);
490 "Time step took " + std::to_string(solverTimer_->secsSinceStart()) +
" seconds; "
491 "total solver time " + std::to_string(report_.success.solver_time) +
" seconds.";
500 SimulatorReport finalize()
504 Dune::Timer finalOutputTimer;
505 finalOutputTimer.start();
507 ebosSimulator_.problem().finalizeOutput();
508 report_.success.output_write_time += finalOutputTimer.stop();
513 report_.success.total_time = totalTimer_->secsSinceStart();
514 report_.success.converged =
true;
519 const Grid& grid()
const
520 {
return ebosSimulator_.vanguard().grid(); }
522 template<
class Serializer>
523 void serializeOp(Serializer& serializer)
525 serializer(ebosSimulator_);
527 serializer(adaptiveTimeStepping_);
530 const Model& model()
const
531 {
return solver_->model(); }
535 std::unique_ptr<Solver> createSolver(WellModel& wellModel)
537 auto model = std::make_unique<Model>(ebosSimulator_,
542 return std::make_unique<Solver>(solverParam_, std::move(model));
545 const EclipseState& eclState()
const
546 {
return ebosSimulator_.vanguard().eclState(); }
549 const Schedule& schedule()
const
550 {
return ebosSimulator_.vanguard().schedule(); }
552 bool isRestart()
const
554 const auto& initconfig = eclState().getInitConfig();
555 return initconfig.restartRequested();
558 WellModel& wellModel_()
559 {
return ebosSimulator_.problem().wellModel(); }
561 const WellModel& wellModel_()
const
562 {
return ebosSimulator_.problem().wellModel(); }
564 void startConvergenceOutputThread(std::string_view convOutputOptions,
565 std::string_view optionName)
567 const auto config = ConvergenceOutputConfiguration {
568 convOutputOptions, optionName
570 if (! config.want(ConvergenceOutputConfiguration::Option::Iterations)) {
575 [compNames =
typename Model::ComponentName{}](
const int compIdx)
576 {
return std::string_view { compNames.name(compIdx) }; }
580 [usys = this->eclState().getUnits()](
const double time)
581 {
return usys.from_si(UnitSystem::measure::time, time); }
584 this->convergenceOutputQueue_.emplace();
585 this->convergenceOutputObject_.emplace
586 (this->eclState().getIOConfig().getOutputDir(),
587 this->eclState().getIOConfig().getBaseName(),
588 std::move(getPhaseName),
589 std::move(convertTime),
590 config, *this->convergenceOutputQueue_);
592 this->convergenceOutputThread_
594 &this->convergenceOutputObject_.value());
597 void writeConvergenceOutput(std::vector<StepReport>&& reports)
599 if (! this->convergenceOutputThread_.has_value()) {
603 auto requests = std::vector<ConvergenceReportQueue::OutputRequest>{};
604 requests.reserve(reports.size());
606 for (
auto&& report : reports) {
607 requests.push_back({ report.report_step, report.current_step, std::move(report.report) });
610 this->convergenceOutputQueue_->enqueue(std::move(requests));
613 void endConvergenceOutputThread()
615 if (! this->convergenceOutputThread_.has_value()) {
619 this->convergenceOutputQueue_->signalLastOutputRequest();
620 this->convergenceOutputThread_->join();
630 OPM_BEGIN_PARALLEL_TRY_CATCH();
636 OpmLog::error(
"Saving of serialized state requested, but no HDF5 support available.");
644 EclGenericVanguard::comm());
646 std::ostringstream
str;
647 Parameters::printValues<TypeTag>(
str);
648 writer.writeHeader(
"OPM Flow",
651 ebosSimulator_.vanguard().caseName(),
653 EclGenericVanguard::comm().size());
655 if (EclGenericVanguard::comm().size() > 1) {
656 const auto&
cellMapping = ebosSimulator_.vanguard().globalCell();
664 OpmLog::info(
"Serialized state written for report step " + std::to_string(
nextStep));
668 OPM_END_PARALLEL_TRY_CATCH(
"Error saving serialized state: ",
669 EclGenericVanguard::comm());
676 OpmLog::error(
"Loading of serialized state requested, but no HDF5 support available.");
679 OPM_BEGIN_PARALLEL_TRY_CATCH();
683 EclGenericVanguard::comm());
689 OpmLog::info(
"Loading serialized state for report step " + std::to_string(
loadStep_));
693 std::tuple<std::array<std::string,5>,
int>
header;
697 if (EclGenericVanguard::comm().size() !=
procs) {
698 throw std::runtime_error(
"Number of processes (procs=" +
699 std::to_string(EclGenericVanguard::comm().size()) +
700 ") does not match .OPMRST file (procs=" +
701 std::to_string(
procs) +
")");
704 if (EclGenericVanguard::comm().size() > 1) {
707 const auto&
cellMapping = ebosSimulator_.vanguard().globalCell();
710 throw std::runtime_error(
"Grid hash mismatch, .OPMRST file cannot be used");
714 if (EclGenericVanguard::comm().
rank() == 0) {
715 std::ostringstream
str;
716 Parameters::printValues<TypeTag>(
str);
717 checkSerializedCmdLine(
str.str(),
strings[4]);
720 OPM_END_PARALLEL_TRY_CATCH(
"Error loading serialized state: ",
721 EclGenericVanguard::comm());
729 OPM_BEGIN_PARALLEL_TRY_CATCH();
733 EclGenericVanguard::comm());
737 OPM_END_PARALLEL_TRY_CATCH(
"Error loading serialized state: ",
738 EclGenericVanguard::comm());
743 Simulator& ebosSimulator_;
745 ModelParameters modelParam_;
746 SolverParameters solverParam_;
748 std::unique_ptr<Solver> solver_;
753 bool terminalOutput_;
756 std::size_t already_reported_steps_ = 0;
757 std::unique_ptr<time::StopWatch> solverTimer_;
758 std::unique_ptr<time::StopWatch> totalTimer_;
759 std::unique_ptr<TimeStepper> adaptiveTimeStepping_;
761 std::optional<ConvergenceReportQueue> convergenceOutputQueue_{};
762 std::optional<ConvergenceOutputThread> convergenceOutputObject_{};
763 std::optional<std::thread> convergenceOutputThread_{};