My Project
Loading...
Searching...
No Matches
FlowMainEbos.hpp
1/*
2 Copyright 2013, 2014, 2015 SINTEF ICT, Applied Mathematics.
3 Copyright 2014 Dr. Blatt - HPC-Simulation-Software & Services
4 Copyright 2015 IRIS AS
5 Copyright 2014 STATOIL ASA.
6
7 This file is part of the Open Porous Media project (OPM).
8
9 OPM is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation, either version 3 of the License, or
12 (at your option) any later version.
13
14 OPM is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with OPM. If not, see <http://www.gnu.org/licenses/>.
21*/
22#ifndef OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
24
25#include <opm/simulators/flow/Banners.hpp>
26#include <opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp>
27
28#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
29#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
30#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
31
32#include <opm/models/utils/start.hh>
33
34#if HAVE_DUNE_FEM
35#include <dune/fem/misc/mpimanager.hh>
36#else
37#include <dune/common/parallel/mpihelper.hh>
38#endif
39
40#include <cstddef>
41#include <memory>
42#include <string_view>
43
44namespace Opm::Properties {
45
46template<class TypeTag, class MyTypeTag>
48 using type = UndefinedProperty;
49};
50template<class TypeTag, class MyTypeTag>
52 using type = UndefinedProperty;
53};
54template<class TypeTag, class MyTypeTag>
58
59// TODO: enumeration parameters. we use strings for now.
60template<class TypeTag>
61struct EnableDryRun<TypeTag, TTag::EclFlowProblem> {
62 static constexpr auto value = "auto";
63};
64// Do not merge parallel output files or warn about them
65template<class TypeTag>
66struct EnableLoggingFalloutWarning<TypeTag, TTag::EclFlowProblem> {
67 static constexpr bool value = false;
68};
69template<class TypeTag>
70struct OutputInterval<TypeTag, TTag::EclFlowProblem> {
71 static constexpr int value = 1;
72};
73
74} // namespace Opm::Properties
75
76namespace Opm {
77namespace detail {
78
79void checkAllMPIProcesses();
80
81void mergeParallelLogFiles(std::string_view output_dir,
82 std::string_view deck_filename,
84
85void handleExtraConvergenceOutput(SimulatorReport& report,
86 std::string_view option,
87 std::string_view optionName,
88 std::string_view output_dir,
89 std::string_view base_name);
90
91}
92
93 class Deck;
94
95 // The FlowMain class is the ebos based black-oil simulator.
96 template <class TypeTag>
98 {
99 public:
107
109
110 FlowMainEbos(int argc, char **argv, bool output_cout, bool output_files )
111 : argc_{argc}, argv_{argv},
112 output_cout_{output_cout}, output_files_{output_files}
113 {
114
115 }
116
117 // Read the command line parameters. Throws an exception if something goes wrong.
118 static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
119 {
121 if (!ParamsMeta::registrationOpen()) {
122 // We have already successfully run setupParameters_().
123 // For the dynamically chosen runs (as from the main flow
124 // executable) we must run this function again with the
125 // real typetag to be used, as the first time was with the
126 // "FlowEarlyBird" typetag. However, for the static ones (such
127 // as 'flow_onephase_energy') it has already been run with the
128 // correct typetag.
129 return EXIT_SUCCESS;
130 }
131 // register the flow specific parameters
132 EWOMS_REGISTER_PARAM(TypeTag, std::string, EnableDryRun,
133 "Specify if the simulation ought to be actually run, or just pretended to be");
134 EWOMS_REGISTER_PARAM(TypeTag, int, OutputInterval,
135 "Specify the number of report steps between two consecutive writes of restart data");
136 EWOMS_REGISTER_PARAM(TypeTag, bool, EnableLoggingFalloutWarning,
137 "Developer option to see whether logging was on non-root processors. In that case it will be appended to the *.DBG or *.PRT files");
138
139 Simulator::registerParameters();
140
141 // register the parameters inherited from ebos
142 registerAllParameters_<TypeTag>(/*finalizeRegistration=*/false);
143
144 // hide the parameters unused by flow. TODO: this is a pain to maintain
147
148 // this parameter is actually used in eWoms, but the flow well model
149 // hard-codes the assumption that the intensive quantities cache is enabled,
150 // so flow crashes. Let's hide the parameter for that reason.
152
153 // thermodynamic hints are not implemented/required by the eWoms blackoil
154 // model
156
157 // in flow only the deck file determines the end time of the simulation
158 EWOMS_HIDE_PARAM(TypeTag, EndTime);
159
160 // time stepping is not done by the eWoms code in flow
166
168
169 // flow also does not use the eWoms Newton method
180
181 // the default eWoms checkpoint/restart mechanism does not work with flow
184 // hide all vtk related it is not currently possible to do this dependet on if the vtk writing is used
185 //if(not(EWOMS_GET_PARAM(TypeTag,bool,EnableVtkOutput))){
213 //}
222
226
227 // hide average density option
228 EWOMS_HIDE_PARAM(TypeTag, UseAverageDensityMsWells);
229
231
232 int mpiRank = comm.rank();
233
234 // read in the command line parameters
235 int status = ::Opm::setupParameters_<TypeTag>(argc, const_cast<const char**>(argv), /*doRegistration=*/false, /*allowUnused=*/true, /*handleHelp=*/(mpiRank==0));
236 if (status == 0) {
237
238 // deal with unknown parameters.
239
240 int unknownKeyWords = 0;
241 if (mpiRank == 0) {
242 unknownKeyWords = Parameters::printUnused<TypeTag>(std::cerr);
243 }
246 if ( unknownKeyWords )
247 {
248 if ( mpiRank == 0 )
249 {
250 std::string msg = "Aborting simulation due to unknown "
251 "parameters. Please query \"flow --help\" for "
252 "supported command line parameters.";
253 if (OpmLog::hasBackend("STREAMLOG"))
254 {
255 OpmLog::error(msg);
256 }
257 else {
258 std::cerr << msg << std::endl;
259 }
260 }
261 return EXIT_FAILURE;
262 }
263
264 // deal with --print-properties and --print-parameters and unknown parameters.
265
266 bool doExit = false;
267
268 if (EWOMS_GET_PARAM(TypeTag, int, PrintProperties) == 1) {
269 doExit = true;
270 if (mpiRank == 0)
271 Properties::printValues<TypeTag>(std::cout);
272 }
273
274 if (EWOMS_GET_PARAM(TypeTag, int, PrintParameters) == 1) {
275 doExit = true;
276 if (mpiRank == 0)
277 Parameters::printValues<TypeTag>();
278 }
279
280 if (doExit)
281 return -1;
282 }
283
284 return status;
285 }
286
291 {
292 return execute_(&FlowMainEbos::runSimulator, /*cleanup=*/true);
293 }
294
295 int executeInitStep()
296 {
297 return execute_(&FlowMainEbos::runSimulatorInit, /*cleanup=*/false);
298 }
299
300 // Returns true unless "EXIT" was encountered in the schedule
301 // section of the input datafile.
302 int executeStep()
303 {
304 return simulator_->runStep(*simtimer_);
305 }
306
307 // Called from Python to cleanup after having executed the last
308 // executeStep()
309 int executeStepsCleanup()
310 {
311 SimulatorReport report = simulator_->finalize();
312 runSimulatorAfterSim_(report);
313 return report.success.exit_status;
314 }
315
316 EbosSimulator *getSimulatorPtr() {
317 return ebosSimulator_.get();
318 }
319
320 SimulatorTimer* getSimTimer() {
321 return simtimer_.get();
322 }
323
324 private:
325 // called by execute() or executeInitStep()
326 int execute_(int (FlowMainEbos::* runOrInitFunc)(), bool cleanup)
327 {
328 auto logger = [this](const std::exception& e, const std::string& message_start) {
329 std::ostringstream message;
330 message << message_start << e.what();
331
332 if (this->output_cout_) {
333 // in some cases exceptions are thrown before the logging system is set
334 // up.
335 if (OpmLog::hasBackend("STREAMLOG")) {
336 OpmLog::error(message.str());
337 }
338 else {
339 std::cout << message.str() << "\n";
340 }
341 }
342 detail::checkAllMPIProcesses();
343 return EXIT_FAILURE;
344 };
345
346 try {
347 // deal with some administrative boilerplate
348
349 int status = setupParameters_(this->argc_, this->argv_, EclGenericVanguard::comm());
350 if (status)
351 return status;
352
353 setupParallelism();
354 setupEbosSimulator();
356
357 // if run, do the actual work, else just initialize
358 int exitCode = (this->*runOrInitFunc)();
359 if (cleanup) {
360 executeCleanup_();
361 }
362 return exitCode;
363 }
364 catch (const TimeSteppingBreakdown& e) {
365 auto exitCode = logger(e, "Simulation aborted: ");
366 executeCleanup_();
367 return exitCode;
368 }
369 catch (const std::exception& e) {
370 auto exitCode = logger(e, "Simulation aborted as program threw an unexpected exception: ");
371 executeCleanup_();
372 return exitCode;
373 }
374 }
375
376 void executeCleanup_() {
377 // clean up
378 mergeParallelLogFiles();
379 }
380
381 protected:
382 void setupParallelism()
383 {
384 // determine the rank of the current process and the number of processes
385 // involved in the simulation. MPI must have already been initialized
386 // here. (yes, the name of this method is misleading.)
387 auto comm = EclGenericVanguard::comm();
388 mpi_rank_ = comm.rank();
389 mpi_size_ = comm.size();
390
391#if _OPENMP
392 // if openMP is available, default to 2 threads per process unless
393 // OMP_NUM_THREADS is set or command line --threads-per-process used
394 if (!getenv("OMP_NUM_THREADS"))
395 {
396 int threads = 2;
397 const int requested_threads = EWOMS_GET_PARAM(TypeTag, int, ThreadsPerProcess);
398 if (requested_threads > 0)
399 threads = requested_threads;
400
401 // We are not limiting this to the number of processes
402 // reported by OpenMP as on some hardware (and some OpenMPI
403 // versions) this will be 1 when run with mpirun
404 omp_set_num_threads(threads);
405 }
406#endif
407
408 using ThreadManager = GetPropType<TypeTag, Properties::ThreadManager>;
409 ThreadManager::init(false);
410 }
411
412 void mergeParallelLogFiles()
413 {
414 // force closing of all log files.
415 OpmLog::removeAllBackends();
416
417 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_ || !ebosSimulator_) {
418 return;
419 }
420
421 detail::mergeParallelLogFiles(eclState().getIOConfig().getOutputDir(),
422 EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName),
423 EWOMS_GET_PARAM(TypeTag, bool, EnableLoggingFalloutWarning));
424 }
425
426 void setupEbosSimulator()
427 {
428 ebosSimulator_ = std::make_unique<EbosSimulator>(EclGenericVanguard::comm(), /*verbose=*/false);
429 ebosSimulator_->executionTimer().start();
430 ebosSimulator_->model().applyInitialSolution();
431
432 try {
433 // Possible to force initialization only behavior (NOSIM).
434 const std::string& dryRunString = EWOMS_GET_PARAM(TypeTag, std::string, EnableDryRun);
435 if (dryRunString != "" && dryRunString != "auto") {
436 bool yesno;
437 if (dryRunString == "true"
438 || dryRunString == "t"
439 || dryRunString == "1")
440 yesno = true;
441 else if (dryRunString == "false"
442 || dryRunString == "f"
443 || dryRunString == "0")
444 yesno = false;
445 else
446 throw std::invalid_argument("Invalid value for parameter EnableDryRun: '"
447 +dryRunString+"'");
448 auto& ioConfig = eclState().getIOConfig();
449 ioConfig.overrideNOSIM(yesno);
450 }
451 }
452 catch (const std::invalid_argument& e) {
453 std::cerr << "Failed to create valid EclipseState object" << std::endl;
454 std::cerr << "Exception caught: " << e.what() << std::endl;
455 throw;
456 }
457 }
458
459 const EclipseState& eclState() const
460 { return ebosSimulator_->vanguard().eclState(); }
461
462 EclipseState& eclState()
463 { return ebosSimulator_->vanguard().eclState(); }
464
465 const Schedule& schedule() const
466 { return ebosSimulator_->vanguard().schedule(); }
467
468 // Run the simulator.
469 int runSimulator()
470 {
471 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorRunCallback_);
472 }
473
474 int runSimulatorInit()
475 {
476 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorInitCallback_);
477 }
478
479 private:
480 // Callback that will be called from runSimulatorInitOrRun_().
481 int runSimulatorRunCallback_()
482 {
483 SimulatorReport report = simulator_->run(*simtimer_);
484 runSimulatorAfterSim_(report);
485 return report.success.exit_status;
486 }
487
488 // Callback that will be called from runSimulatorInitOrRun_().
489 int runSimulatorInitCallback_()
490 {
491 simulator_->init(*simtimer_);
492 return EXIT_SUCCESS;
493 }
494
495 // Output summary after simulation has completed
496 void runSimulatorAfterSim_(SimulatorReport &report)
497 {
498 if (! this->output_cout_) {
499 return;
500 }
501
502 const int threads
503#if !defined(_OPENMP) || !_OPENMP
504 = 1;
505#else
506 = omp_get_max_threads();
507#endif
508
509 printFlowTrailer(mpi_size_, threads, report, simulator_->model().localAccumulatedReports());
510
511 detail::handleExtraConvergenceOutput(report,
512 EWOMS_GET_PARAM(TypeTag, std::string, OutputExtraConvergenceInfo),
513 R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
514 eclState().getIOConfig().getOutputDir(),
515 eclState().getIOConfig().getBaseName());
516 }
517
518 // Run the simulator.
519 int runSimulatorInitOrRun_(int (FlowMainEbos::* initOrRunFunc)())
520 {
521
522 const auto& schedule = this->schedule();
523 auto& ioConfig = eclState().getIOConfig();
524 simtimer_ = std::make_unique<SimulatorTimer>();
525
526 // initialize variables
527 const auto& initConfig = eclState().getInitConfig();
528 simtimer_->init(schedule, static_cast<std::size_t>(initConfig.getRestartStep()));
529
530 if (this->output_cout_) {
531 std::ostringstream oss;
532
533 // This allows a user to catch typos and misunderstandings in the
534 // use of simulator parameters.
535 if (Parameters::printUnused<TypeTag>(oss)) {
536 std::cout << "----------------- Unrecognized parameters: -----------------\n";
537 std::cout << oss.str();
538 std::cout << "----------------------------------------------------------------" << std::endl;
539 }
540 }
541
542 if (!ioConfig.initOnly()) {
543 if (this->output_cout_) {
544 std::string msg;
545 msg = "\n\n================ Starting main simulation loop ===============\n";
546 OpmLog::info(msg);
547 }
548
549 return (this->*initOrRunFunc)();
550 }
551 else {
552 if (this->output_cout_) {
553 std::cout << "\n\n================ Simulation turned off ===============\n" << std::flush;
554 }
555 return EXIT_SUCCESS;
556 }
557 }
558
559 protected:
560
562 // Create simulator instance.
563 // Writes to:
564 // simulator_
566 {
567 // Create the simulator instance.
568 simulator_ = std::make_unique<Simulator>(*ebosSimulator_);
569 }
570
571 Grid& grid()
572 { return ebosSimulator_->vanguard().grid(); }
573
574 private:
575 std::unique_ptr<EbosSimulator> ebosSimulator_;
576 int mpi_rank_ = 0;
577 int mpi_size_ = 1;
578 std::any parallel_information_;
579 std::unique_ptr<Simulator> simulator_;
580 std::unique_ptr<SimulatorTimer> simtimer_;
581 int argc_;
582 char **argv_;
583 bool output_cout_;
584 bool output_files_;
585 };
586
587} // namespace Opm
588
589#endif // OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
Definition AquiferInterface.hpp:35
Definition FlowMainEbos.hpp:98
int execute()
This is the main function of Flow.
Definition FlowMainEbos.hpp:290
void createSimulator()
This is the main function of Flow.
Definition FlowMainEbos.hpp:565
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition BlackoilPhases.hpp:27
Definition FlowMainEbos.hpp:47
Definition FlowMainEbos.hpp:55
Definition FlowMainEbos.hpp:51
Definition SimulatorReport.hpp:100