diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 3544f028267..bbbc5b8a516 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -5641,14 +5641,14 @@ class CConfig { unsigned short GetnVar(void); /*! - * \brief Provides the number of variables. - * \return Number of variables. + * \brief Provides the total number of zones. + * \return Total number of zones. */ unsigned short GetnZone(void) const { return nZone; } /*! - * \brief Provides the number of variables. - * \return Number of variables. + * \brief Provides the zone index the configuration belongs to. + * \return Zone index. */ unsigned short GetiZone(void) const { return iZone; } diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 2945e4f9ef8..e6e58afb189 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -1,7 +1,7 @@ /*! * \file ad_structure.hpp * \brief Main routines for the algorithmic differentiation (AD) structure. - * \author T. Albring, J. Blühdorn + * \author T. Albring, J. Blühdorn, O. Burghardt * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -36,7 +36,21 @@ * In case there is no reverse type configured, they have no effect at all, * and so the real versions of the routined are after #else. */ + namespace AD { + +enum class TAPE_DEBUG_OPTION { + ALLOW_PREACC, + ACTIVATE_PREACC, + ALLOW_ZONE, + ALLOW_ALL_ZONES, + ACTIVATE_ALL_ZONES, + ACTIVATE_ALL_ERRORS, + MULTIZONE_TAGS, + INIT_RUN, + CHECK_RUN +}; + #ifndef CODI_REVERSE_TYPE using Identifier = int; @@ -287,42 +301,73 @@ inline void SetIndex(Identifier& index, const su2double& data) {} */ inline void SetTag(int tag) {} +/*! + * \brief Gets the current tag. + * \param[in] tag - the number to which the tag is set. + */ +inline int GetTag() { return 0; } + +/*! + * \brief Compute the zone-specific tag. + * \param[in] iZone - Zone index from which the zone-specific tag is formed. + * \return The zone-specific tag. + */ +inline int ComputeTag(unsigned short iZone) { return 0; } + /*! * \brief Sets the tag of a variable to 0. * \param[in] v - the variable whose tag is cleared. */ inline void ClearTagOnVariable(su2double& v) {} +/*! + * \brief Sets the tag of a variable to a specified value. + * \param[in] v - the variable whose tag is set manually. + */ +inline void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) {} + /*! * \brief Struct to store information about errors during a tag debug run. */ -struct ErrorReport {}; +struct DebugControl {}; /*! - * \brief Set a reference to the output file of an ErrorReport. - * \param[in] report - the ErrorReport whose output file is set. - * \param[in] output_file - pointer to the output file. + * \brief Set a pointer to the current DebugControl. + * \param[in] control - pointer to the current debug control. */ -inline void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) {} +inline void SetDebugControl(DebugControl* control) {} /*! - * \brief Set the ErrorReport to which error information from a tag debug recording is written. - * \param[in] report - the ErrorReport to which error information is written. + * \brief Set options which kind of tag mismatches are considered errors and written to file. + * \param[in] option - specification which kind of tag mismatches are considered. + * \param[in] izone - the zone w.r.t. which a tag mismatch is allowed. */ -inline void SetTagErrorCallback(ErrorReport& report) {} +inline void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) {} /*! - * \brief Reset the error counter in an ErrorReport. - * \param[in] report - the ErrorReport whose error counter is resetted. + * \brief Activate the error callback by letting the tape call tagErrorCallback whenever a tag mismatch arises. */ -inline void ResetErrorCounter(ErrorReport& report) {} +inline void ActivateTagErrorCallback() {} /*! - * \brief Get the error count of an ErrorReport. - * \param[in] report - the ErrorReport whose pointer to its error counter is returned. + * \brief Set a pointer to the output file of a DebugControl. + * \param[in] control - the DebugControl whose output file is set. + * \param[in] output_file - pointer to the output file. + */ +inline void SetDebugReportFile(DebugControl& control, std::ostream* output_file) {} + +/*! + * \brief Reset the error counter in a DebugControl. + * \param[in] control - the DebugControl whose error counter is resetted. + */ +inline void ResetErrorCounter(DebugControl& control) {} + +/*! + * \brief Get the error count of a DebugControl. + * \param[in] control - the DebugControl whose error count is reported. * \return Value of the error counter. */ -inline unsigned long GetErrorCount(const ErrorReport& report) { return 0; } +inline unsigned long GetErrorCount(const DebugControl& control) { return 0; } /*! * \brief Pushes back the current tape position to the tape position's vector. @@ -744,37 +789,142 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } -struct ErrorReport { +#ifdef CODI_TAG_TAPE + +struct DebugControl { + int current_tag; + bool multizone_tags = false; + bool init_run = false; + bool allow_preacc = false; + bool allow_zones = false; + unsigned short allow_izone = 0; unsigned long ErrorCounter = 0; std::ostream* out = &std::cout; }; -FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } +struct AdjustDebugControl { + TAPE_DEBUG_OPTION option; + void (*adjust)(DebugControl*, unsigned short); +}; -FORCEINLINE void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) { report.out = output_file; } +static const AdjustDebugControl debug_control_adjustments[] = { + {TAPE_DEBUG_OPTION::ALLOW_PREACC, [](DebugControl* c, unsigned short) { c->allow_preacc = true; }}, + {TAPE_DEBUG_OPTION::ACTIVATE_PREACC, [](DebugControl* c, unsigned short) { c->allow_preacc = false; }}, + {TAPE_DEBUG_OPTION::ALLOW_ZONE, [](DebugControl* c, unsigned short zone) { c->allow_izone = zone + 1; }}, + {TAPE_DEBUG_OPTION::ALLOW_ALL_ZONES, [](DebugControl* c, unsigned short) { c->allow_zones = true; }}, + {TAPE_DEBUG_OPTION::ACTIVATE_ALL_ZONES, + [](DebugControl* c, unsigned short) { + c->allow_zones = false; + c->allow_izone = 0; + }}, + {TAPE_DEBUG_OPTION::ACTIVATE_ALL_ERRORS, + [](DebugControl* c, unsigned short) { + c->allow_preacc = false; + c->allow_zones = false; + c->allow_izone = 0; + }}, + {TAPE_DEBUG_OPTION::INIT_RUN, [](DebugControl* c, unsigned short) { c->init_run = true; }}, + {TAPE_DEBUG_OPTION::CHECK_RUN, [](DebugControl* c, unsigned short) { c->init_run = false; }}, + {TAPE_DEBUG_OPTION::MULTIZONE_TAGS, [](DebugControl* c, unsigned short) { c->multizone_tags = true; }}, +}; -FORCEINLINE unsigned long GetErrorCount(const ErrorReport& report) { return report.ErrorCounter; } +FORCEINLINE void ResetErrorCounter(DebugControl& control) { control.ErrorCounter = 0; } -#ifdef CODI_TAG_TAPE +FORCEINLINE void SetDebugReportFile(DebugControl& control, std::ostream* output_file) { control.out = output_file; } + +FORCEINLINE unsigned long GetErrorCount(const DebugControl& control) { return control.ErrorCounter; } + +extern DebugControl* current_control; + +FORCEINLINE void SetDebugControl(DebugControl* control) { current_control = control; } + +FORCEINLINE void SetTag(int tag) { + current_control->current_tag = tag; + AD::getTape().setCurTag(tag); +} + +FORCEINLINE int GetTag() { return current_control->current_tag; } + +FORCEINLINE int ComputeTag(unsigned short iZone) { + if (current_control->init_run) { + return (current_control->multizone_tags) ? ((int)iZone + 1) * 10 + 1 : 1; + } else { + return (current_control->multizone_tags) ? ((int)iZone + 1) * 10 + 2 : 2; + } +} -FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } +FORCEINLINE void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) { + int tag = v.getIdentifier().tag; + int tens = (zone_tag > 0) ? zone_tag + 1 : tag / 10; + int ones = (run_tag > 0 && run_tag < 10) ? run_tag : tag % 10; + if (tag != 0) { + v.getIdentifier().tag = tens * 10 + ones; + } +} + static void tagErrorCallback(const int& correctTag, const int& wrongTag, void* userData) { - auto* report = static_cast(userData); + auto* status = static_cast(userData); + + bool throw_mismatch_error = true; - report->ErrorCounter += 1; - *(report->out) << "Use of variable with bad tag '" << wrongTag << "', should be '" << correctTag << "'." << std::endl; + /*--- The callback could be due to a preaccumulation tag mismatch that we maybe want to allow, ... ---*/ + if (status->allow_preacc) { + if (correctTag == 1337) { + throw_mismatch_error = false; + } + } + + /*--- ... or to a mismatch in the zone part of the tag. ---*/ + if (status->allow_zones) { + throw_mismatch_error = false; + } else if (status->allow_izone > 0) { + if (wrongTag / 10 == status->allow_izone) { + throw_mismatch_error = false; + } + } + + if (throw_mismatch_error) { + status->ErrorCounter += 1; + *(status->out) << "Use of variable with bad tag '" << std::setw(2) << std::setfill('0') << wrongTag + << "', should be '" << std::setw(2) << std::setfill('0') << correctTag << "'." << std::endl; + } } -FORCEINLINE void SetTagErrorCallback(ErrorReport& report) { - AD::getTape().setTagErrorCallback(tagErrorCallback, &report); +FORCEINLINE void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) { + if (current_control == nullptr) { + return; + } + for (const AdjustDebugControl& entry : debug_control_adjustments) { + if (entry.option == option) { + entry.adjust(current_control, izone); + break; + } + } +} + +FORCEINLINE void ActivateTagErrorCallback() { + if (current_control != NULL) { + AD::getTape().setTagErrorCallback(tagErrorCallback, current_control); + } else { + std::cout << "No tape debug control set!" << std::endl; + } } #else +struct DebugControl {}; +FORCEINLINE void ResetErrorCounter(DebugControl& control) {} +FORCEINLINE void SetDebugReportFile(DebugControl& control, std::ostream* output_file) {} +FORCEINLINE unsigned long GetErrorCount(const DebugControl& control) { return 0; } +FORCEINLINE void SetDebugControl(DebugControl* status) {} +FORCEINLINE int GetTag() { return 0; } FORCEINLINE void SetTag(int tag) {} +FORCEINLINE int ComputeTag(unsigned short iZone) { return 0; } FORCEINLINE void ClearTagOnVariable(su2double& v) {} -FORCEINLINE void SetTagErrorCallback(ErrorReport report) {} +FORCEINLINE void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) {} +FORCEINLINE void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) {} +FORCEINLINE void ActivateTagErrorCallback() {} #endif // CODI_TAG_TAPE diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index d4d0e7f0fa0..3d5d57c1767 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2701,11 +2701,7 @@ enum class RECORDING { SOLUTION_VARIABLES, MESH_COORDS, MESH_DEFORM, - SOLUTION_AND_MESH, - TAG_INIT_SOLVER_VARIABLES, - TAG_CHECK_SOLVER_VARIABLES, - TAG_INIT_SOLVER_AND_MESH, - TAG_CHECK_SOLVER_AND_MESH + SOLUTION_AND_MESH }; /*! diff --git a/Common/src/basic_types/ad_structure.cpp b/Common/src/basic_types/ad_structure.cpp index d22ea67f373..2e122484797 100644 --- a/Common/src/basic_types/ad_structure.cpp +++ b/Common/src/basic_types/ad_structure.cpp @@ -48,6 +48,10 @@ SU2_OMP(threadprivate(PreaccHelper)) ExtFuncHelper FuncHelper; +#ifdef CODI_TAG_TAPE +DebugControl* current_control = NULL; +#endif // CODI_TAG_TAPE + #endif void Initialize() { diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 5fe657d48f5..ac3bc15e713 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -117,16 +117,6 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void StartSolver() override; - /*! - * \brief [Overload] Launch the tape test mode for the discrete adjoint multizone solver. - */ - void TapeTest (); - - /*! - * \brief [Overload] Get error numbers after a tape test run of the discrete adjoint multizone solver. - */ - int TapeTestGatherErrors(AD::ErrorReport& error_report) const; - /*! * \brief Preprocess the multizone iteration */ @@ -279,4 +269,15 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { } } + /*! + * \brief Launch the tape test run of the discrete adjoint multizone solver. + */ + void TapeTest (); + + /*! + * \brief Get the total error count after a tape test run of the discrete adjoint multizone solver. + * \param[in] debug_control - DebugControl from which this rank's contribution to the total error count is read. + * \return The total error count across all ranks. + */ + int TapeTestGatherErrors(AD::DebugControl& debug_control) const; }; diff --git a/SU2_CFD/include/output/CFlowOutput.hpp b/SU2_CFD/include/output/CFlowOutput.hpp index cf4b2c9f6aa..7e6264f1847 100644 --- a/SU2_CFD/include/output/CFlowOutput.hpp +++ b/SU2_CFD/include/output/CFlowOutput.hpp @@ -116,6 +116,14 @@ class CFlowOutput : public CFVMOutput{ */ void LoadHistoryDataScalar(const CConfig* config, const CSolver* const* solver); + /*! + * \brief Recompute history output field values that can be used as objective functions in the (multiphysics) discrete adjoint solver. + * \param[in] config - Definition of the particular problem. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - The container holding all solution data. + */ + void LoadCustomAndComboObjectiveFunctions(CConfig *config, CGeometry *geometry, CSolver **solver) override; + /*! * \brief Add scalar (turbulence/species) volume solution fields for a point (FVMComp, FVMInc, FVMNEMO). * \note The order of fields in restart files is fixed. Therefore the split-up. diff --git a/SU2_CFD/include/output/COutput.hpp b/SU2_CFD/include/output/COutput.hpp index a1fea3fdf46..ea47413ae8a 100644 --- a/SU2_CFD/include/output/COutput.hpp +++ b/SU2_CFD/include/output/COutput.hpp @@ -427,6 +427,15 @@ class COutput { void SetMultizoneHistoryOutput(COutput** output, CConfig **config, CConfig *driver_config, unsigned long TimeIter, unsigned long OuterIter); + /*! + * \brief Evaluates objective functions in the (multiphysics) discrete adjoint solver. + * \note Uses the same subroutines for objective function evaluation as SetHistoryOutput, but omits unnecessary evaluations (e.g. residuals, convergence data) to avoid AD complications. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + */ + void SetObjectiveFunctionValues(CGeometry *geometry, CSolver **solver_container, CConfig *config); + /*! * \brief Sets the volume output filename * \param[in] filename - the new filename @@ -996,6 +1005,17 @@ class COutput { */ inline virtual void LoadHistoryData(CConfig *config, CGeometry *geometry, CSolver **solver) {} + /*! + * \brief Recompute history output field values that can be used as objective functions in the (multiphysics) discrete adjoint solver. + * \param[in] config - Definition of the particular problem. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - The container holding all solution data. + */ + inline virtual void LoadCustomAndComboObjectiveFunctions(CConfig *config, CGeometry *geometry, CSolver **solver) { + /*--- Unless LoadCustomAndComboObjectiveFunctions is implemented in a derived output class, we use LoadHistoryData (not ideal for AD). ---*/ + LoadHistoryData(config, geometry, solver); + } + /*! * \brief Load the multizone history output field values * \param[in] output - Container holding the output instances per zone. diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 08c9fce72ba..47ed928cb87 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -269,22 +269,48 @@ void CDiscAdjMultizoneDriver::StartSolver() { void CDiscAdjMultizoneDriver::TapeTest() { SU2_ZONE_SCOPED + if (nZone > 100) { + SU2_MPI::Error("The tape debug mode tag system is limited to a maximum zone number of 100.", CURRENT_FUNCTION); + } + if (rank == MASTER_NODE) { cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; } int total_errors = 0; - AD::ErrorReport error_report; - AD::SetTagErrorCallback(error_report); - std::ofstream out1("run1_process" + to_string(rank) + ".out"); - std::ofstream out2("run2_process" + to_string(rank) + ".out"); - AD::ResetErrorCounter(error_report); - AD::SetDebugReportFile(error_report, &out1); + /*--- Errors are reported to an instance of AD::DebugControl that holds an error counter, a pointer to + * an error log file and configurations determined by the TAPE_DEBUG_OPTION settings. ---*/ + AD::DebugControl debug_control; + + /*--- Set a pointer to the current status internally in the AD structure. ---*/ + AD::SetDebugControl(&debug_control); + + /*--- Set the callback function that handles the event of a tag mismatch on the tape. ---*/ + AD::ActivateTagErrorCallback(); + + /*--- For multizone cases (nZone > 1), we use zone-specific tags. ---*/ + if(nZone > 1) { AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::MULTIZONE_TAGS); } + + /*--- Set the default tag mismatch callback (consider every mismatch an error). ---*/ + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::ACTIVATE_ALL_ERRORS); - /*--- This recording will assign the initial (same) tag to each registered variable. + // Make this a config option? + // AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::ALLOW_ALL_ZONES); + + /*--- Reset the error counter and set the error log file (each process writes its own). ---*/ + AD::ResetErrorCounter(debug_control); + std::ofstream out("debug_run_process" + to_string(rank) + ".out"); + AD::SetDebugReportFile(debug_control, &out); + + /*--- This recording will assign an initial, zone-specific tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ - AD::SetTag(1); + + out << "-----------------------------------------------------------------------------------------" << std::endl; + out << "INITIAL recording." << std::endl; + out << "Errors appearing in this recording are most likely preaccumulation errors (preaccumulation tag: 1337).\n" << std::endl; + + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::INIT_RUN); if(driver_config->GetAD_CheckTapeType() == CHECK_TAPE_TYPE::OBJECTIVE_FUNCTION) { if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) { @@ -292,7 +318,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { SetRecording(RECORDING::MESH_COORDS, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } else { - if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLUTION_VARIABLES." << endl; + if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES." << endl; SetRecording(RECORDING::SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } } @@ -302,21 +328,29 @@ void CDiscAdjMultizoneDriver::TapeTest() { SetRecording(RECORDING::MESH_COORDS, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } else { - if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLUTION_VARIABLES." << endl; + if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLVER_VARIABLES." << endl; SetRecording(RECORDING::SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } } - total_errors = TapeTestGatherErrors(error_report); - AD::ResetErrorCounter(error_report); - AD::SetDebugReportFile(error_report, &out2); + /*--- Gather errors from all ranks from the initial recording (e.g. preaccumulation errors). ---*/ + total_errors = TapeTestGatherErrors(debug_control); + AD::ResetErrorCounter(debug_control); /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag * from the initial recording and a mismatch with the "check" recording tag will throw an error. * In such a case, a possible reason could be that such a variable is set by a post-processing routine while * for a mathematically correct recording this dependency must be included earlier. ---*/ - AD::SetTag(2); + + out << "-------------------------------------------------------------------------------------------------" << std::endl; + out << "IZONE = " << iZone << ", SECOND recording." << std::endl; + out << "Errors appearing hereafter are most likely mathematical errors (e.g. check for circular dependencies)." << std::endl; + + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::CHECK_RUN); + + /*--- We ignore preaccumulation mismatches during the second recording as they have already been reported. ---*/ + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::ALLOW_PREACC); if(driver_config->GetAD_CheckTapeType() == CHECK_TAPE_TYPE::OBJECTIVE_FUNCTION) { if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) @@ -330,7 +364,8 @@ void CDiscAdjMultizoneDriver::TapeTest() { else SetRecording(RECORDING::SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } - total_errors += TapeTestGatherErrors(error_report); + total_errors += TapeTestGatherErrors(debug_control); + if (rank == MASTER_NODE) { cout << "\n------------------------- Tape Test Run Summary -------------------------" << endl; @@ -339,10 +374,10 @@ void CDiscAdjMultizoneDriver::TapeTest() { } } -int CDiscAdjMultizoneDriver::TapeTestGatherErrors(AD::ErrorReport& error_report) const { +int CDiscAdjMultizoneDriver::TapeTestGatherErrors(AD::DebugControl& debug_control) const { SU2_ZONE_SCOPED - int num_errors = AD::GetErrorCount(error_report); + int num_errors = AD::GetErrorCount(debug_control); int total_errors = 0; std::vector process_error(size); SU2_MPI::Allreduce(&num_errors, &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); @@ -757,10 +792,6 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES.\n"; cout << "Computing residuals to check the convergence of the direct problem." << endl; break; - case RECORDING::TAG_INIT_SOLVER_VARIABLES: cout << "Simulating recording with tag 1 on conservative variables." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_SOLVER_VARIABLES: cout << "Checking first recording with tag 2 on conservative variables." << endl; AD::SetTag(2); break; - case RECORDING::TAG_INIT_SOLVER_AND_MESH: cout << "Simulating recording with tag 1 on conservative variables and mesh coordinates." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_SOLVER_AND_MESH: cout << "Checking first recording with tag 2 on conservative variables and mesh coordinates." << endl; AD::SetTag(2); break; default: break; } } @@ -785,6 +816,12 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t type_recording = RECORDING::MESH_DEFORM; } + /*--- If we are in tape debug mode, set a global (but zone-specific) tag. + * Every variable that we register below will be initialized with this tag. ---*/ + int tag = AD::ComputeTag(iZone); + AD::SetTag(tag); + if(tag != 0) { cout << " - register input variables with tag " << AD::GetTag() << " on zone " << iZone << "." << endl; } + iteration_container[iZone][INST_0]->RegisterInput(solver_container, geometry_container, config_container, iZone, INST_0, type_recording); } @@ -793,6 +830,10 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t AD::Push_TapePosition(); /// REGISTERED for (iZone = 0; iZone < nZone; iZone++) { + + /*--- If in tape debug mode, set the zone-specific tag for computations in this zone. ---*/ + AD::SetTag(AD::ComputeTag(iZone)); + iteration_container[iZone][INST_0]->SetDependencies(solver_container, geometry_container, numerics_container, config_container, iZone, INST_0, kind_recording); } @@ -806,6 +847,10 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t if ((tape_type == Kind_Tape::OBJECTIVE_FUNCTION_TAPE) || (kind_recording == RECORDING::MESH_COORDS)) { HandleDataTransfer(); for (iZone = 0; iZone < nZone; iZone++) { + + /*--- If in tape debug mode, set the zone-specific test tag for computations in this zone. ---*/ + AD::SetTag(AD::ComputeTag(iZone)); + if (Has_Deformation(iZone)) { iteration_container[iZone][INST_0]->SetDependencies(solver_container, geometry_container, numerics_container, config_container, iZone, INST_0, kind_recording); @@ -832,6 +877,11 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t AD::Push_TapePosition(); /// enter_zone + /*--- If in tape debug mode, set the zone-specific test tag for computations in this zone. ---*/ + int tag = AD::ComputeTag(iZone); + AD::SetTag(tag); + if(tag != 0) { cout << " - check solver of zone " << iZone << " against tag " << tag << endl; } + DirectIteration(iZone, kind_recording); iteration_container[iZone][INST_0]->RegisterOutput(solver_container, geometry_container, @@ -897,13 +947,13 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { solvers[FLOW_SOL]->ComputeTurboBladePerformance(geometry, config, iZone); } - direct_output[iZone]->SetHistoryOutput(geometry, solvers, config); + direct_output[iZone]->SetObjectiveFunctionValues(geometry, solvers, config); ObjFunc += solvers[FLOW_SOL]->GetTotal_ComboObj(); break; case MAIN_SOLVER::DISC_ADJ_HEAT: solvers[HEAT_SOL]->Heat_Fluxes(geometry, solvers, config); - direct_output[iZone]->SetHistoryOutput(geometry, solvers, config); + direct_output[iZone]->SetObjectiveFunctionValues(geometry, solvers, config); ObjFunc += solvers[HEAT_SOL]->GetTotal_ComboObj(); break; @@ -912,7 +962,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { solvers[HEAT_SOL]->Heat_Fluxes(geometry, solvers, config); } solvers[FEA_SOL]->Postprocessing(geometry, config, numerics_container[iZone][INST_0][MESH_0][FEA_SOL], true); - direct_output[iZone]->SetHistoryOutput(geometry, solvers, config); + direct_output[iZone]->SetObjectiveFunctionValues(geometry, solvers, config); ObjFunc += solvers[FEA_SOL]->GetTotal_ComboObj(); break; @@ -924,11 +974,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { if (rank == MASTER_NODE) { AD::RegisterOutput(ObjFunc); AD::SetIndex(ObjFunc_Index, ObjFunc); - if (kind_recording == RECORDING::SOLUTION_VARIABLES || - kind_recording == RECORDING::TAG_INIT_SOLVER_VARIABLES || - kind_recording == RECORDING::TAG_CHECK_SOLVER_VARIABLES || - kind_recording == RECORDING::TAG_INIT_SOLVER_AND_MESH || - kind_recording == RECORDING::TAG_CHECK_SOLVER_AND_MESH) { + if (kind_recording == RECORDING::SOLUTION_VARIABLES) { cout << "Objective function value: " << std::setprecision(driver_config->GetOutput_Precision()) << ObjFunc << endl; } } @@ -1035,10 +1081,15 @@ void CDiscAdjMultizoneDriver::HandleDataTransfer() { /*--- In principle, the mesh does not need to be updated ---*/ bool DeformMesh = false; + int tag = AD::ComputeTag(iZone); + AD::SetTag(tag); + if(tag != 0) { cout << " - check data transfer of variables into zone " << iZone << " against its (correct) tag " << tag << endl; } + /*--- Transfer from all the remaining zones ---*/ for (unsigned short jZone = 0; jZone < nZone; jZone++){ /*--- The target zone is iZone ---*/ if (jZone != iZone && interface_container[jZone][iZone] != nullptr) { + if(tag != 0) { cout << " - From zone " << jZone << " into zone " << iZone << endl;} DeformMesh |= TransferData(jZone, iZone); } } diff --git a/SU2_CFD/src/interfaces/CInterface.cpp b/SU2_CFD/src/interfaces/CInterface.cpp index ce189fd15dc..6a2552d45f7 100644 --- a/SU2_CFD/src/interfaces/CInterface.cpp +++ b/SU2_CFD/src/interfaces/CInterface.cpp @@ -121,8 +121,21 @@ void CInterface::BroadcastData(const CInterpolator& interpolator, /*--- If this processor owns the node. ---*/ if (donor_geometry->nodes->GetDomain(iPoint)) { + /*--- Read variables from donor solver. + * If in AD test recording mode, keep the current tag, but allow the donor tag while loading the donor variable into Donor_Variable. ---*/ + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::ALLOW_ZONE, donor_config->GetiZone()); + GetDonor_Variable(donor_solution, donor_geometry, donor_config, markDonor, iVertex, iPoint); - for (auto iVar = 0u; iVar < nVar; iVar++) sendDonorVar(iSend, iVar) = Donor_Variable[iVar]; + + /*--- If in AD test recording mode, we manually adapt the tag to the target (this) zone and return to strict tag mismatch handling. ---*/ + for (auto iVar = 0u; iVar < nVar; iVar++) { + AD::SetTagOnVariable(Donor_Variable[iVar], target_config->GetiZone()); + } + AD::SetTapeDebugOption(AD::TAPE_DEBUG_OPTION::ACTIVATE_ALL_ZONES); + + for (auto iVar = 0u; iVar < nVar; iVar++) { + sendDonorVar(iSend, iVar) = Donor_Variable[iVar]; + } sendDonorIdx[iSend] = donor_geometry->nodes->GetGlobalIndex(iPoint); ++iSend; diff --git a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp index a3bd1d0616e..b1c1fff0937 100644 --- a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp @@ -410,7 +410,8 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge SU2_OMP_PARALLEL_(if(solvers0[ADJFLOW_SOL]->GetHasHybridParallel())) { bool AD_debug_mesh_coordinates = false; - if (config[iZone]->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) { + if(config[iZone]->GetDiscrete_Adjoint_Debug() && + config[iZone]->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) { cout << "Register additional SOLUTION VARIABLES for tag debug mode (zone " << iZone << ")." << endl; AD_debug_mesh_coordinates = true; } diff --git a/SU2_CFD/src/output/CFlowOutput.cpp b/SU2_CFD/src/output/CFlowOutput.cpp index 06f0534e11a..6abf8c9b5a9 100644 --- a/SU2_CFD/src/output/CFlowOutput.cpp +++ b/SU2_CFD/src/output/CFlowOutput.cpp @@ -1405,6 +1405,10 @@ void CFlowOutput::SetVolumeOutputFieldsScalarSolution(const CConfig* config){ } } +void CFlowOutput::LoadCustomAndComboObjectiveFunctions(CConfig *config, CGeometry *geometry, CSolver **solver) { + LoadHistoryData(config, geometry, solver); +} + void CFlowOutput::SetVolumeOutputFieldsScalarResidual(const CConfig* config) { /*--- Only place outputs of the "RESIDUAL" group here. ---*/ diff --git a/SU2_CFD/src/output/COutput.cpp b/SU2_CFD/src/output/COutput.cpp index 07c053d629a..06c4272f9ca 100644 --- a/SU2_CFD/src/output/COutput.cpp +++ b/SU2_CFD/src/output/COutput.cpp @@ -231,6 +231,10 @@ void COutput::SetHistoryOutput(CGeometry *geometry, } +void COutput::SetObjectiveFunctionValues(CGeometry *geometry, CSolver **solver_container, CConfig *config) { + LoadCustomAndComboObjectiveFunctions(config, geometry, solver_container); +} + void COutput::SetHistoryOutput(CGeometry ****geometry, CSolver *****solver, CConfig **config, std::shared_ptr(TurboStagePerf), su2vector> TurboBladePerfs, unsigned short val_iZone, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter, unsigned short val_iInst){ unsigned long Iter= InnerIter;