From dff89cf2deae154608045acf9c85779e0822ec1e Mon Sep 17 00:00:00 2001 From: moneta Date: Wed, 6 Sep 2023 11:03:07 +0200 Subject: [PATCH] [tmva][pymva] Do not disable tensorflow eager execution on MacoS Disbaling it causes an interference with multiprocessing and a restart of the process after it is endeded. This fixes the timeout of the PyKeras tests on MacOS. This commits apply also some small fixes/improvements on the deletetion of some TMVA objects. --- tmva/pymva/src/MethodPyKeras.cxx | 27 +++++++++++++++++---------- tmva/pymva/src/PyMethodBase.cxx | 5 ++++- tmva/tmva/src/DataSetInfo.cxx | 6 +++--- tmva/tmva/src/MethodBase.cxx | 8 ++++---- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/tmva/pymva/src/MethodPyKeras.cxx b/tmva/pymva/src/MethodPyKeras.cxx index def5c7a4e71f1..c915ab70018d4 100644 --- a/tmva/pymva/src/MethodPyKeras.cxx +++ b/tmva/pymva/src/MethodPyKeras.cxx @@ -39,6 +39,7 @@ ClassImp(MethodPyKeras); MethodPyKeras::MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption) : PyMethodBase(jobName, Types::kPyKeras, methodTitle, dsi, theOption) { fNumEpochs = 10; + fNumThreads = 0; fBatchSize = 100; fVerbose = 1; fContinueTraining = false; @@ -380,23 +381,29 @@ void MethodPyKeras::SetupKerasModelForEval() { // disable eager execution (model will evaluate > 100 faster) // need to be done before loading the model +#ifndef R__MACOSX // problem siabling eager execution on Macos (conflict with multiprocessing) if (fUseTFKeras){ PyRunString("tf.compat.v1.disable_eager_execution()","Failed to disable eager execution"); Log() << kINFO << "Disabled TF eager execution when evaluating model " << Endl; } +#endif SetupKerasModel(true); // Init evaluation (needed for getMvaValue) - fVals.resize(fNVars); // holds values used for classification and regression - npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars}; - PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals.data()); - PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals); + if (fNVars > 0) { + fVals.resize(fNVars); // holds values used for classification and regression + npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars}; + PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals.data()); + PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals); + } // setup output variables - fOutput.resize(fNOutputs); // holds classification probabilities or regression output - npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs}; - PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)fOutput.data()); - PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput); + if (fNOutputs > 0) { + fOutput.resize(fNOutputs); // holds classification probabilities or regression output + npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs}; + PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)fOutput.data()); + PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput); + } fModelIsSetupForEval = true; } @@ -707,6 +714,7 @@ std::vector MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t la } } + std::vector mvaValues(nEvents); npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)fNVars}; PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (void*)data); if (pDataMvaValues==0) Log() << "Failed to load data to Python array" << Endl; @@ -717,11 +725,10 @@ std::vector MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t la PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallMethod(pModel, (char*)"predict", (char*)"O", pDataMvaValues); if (pPredictions==0) Log() << kFATAL << "Failed to get predictions" << Endl; delete[] data; - // Load predictions to double vector // NOTE: The signal probability is given at the output - std::vector mvaValues(nEvents); float* predictionsData = (float*) PyArray_DATA(pPredictions); + for (UInt_t i=0; i> " << code << std::endl; fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS); if (!fPyReturn) { Log() << kWARNING << "Failed to run python code: " << code << Endl; @@ -397,7 +400,7 @@ std::vector PyMethodBase::GetDataFromList(PyObject* listObject){ } ////////////////////////////////////////////////////////////////////////////////// -/// \brief Utility function which checks if a given key is present in a Python +/// \brief Utility function which checks if a given key is present in a Python /// dictionary object and returns the associated value or throws runtime /// error. This is to replace PyDict_GetItemWithError in Python 2. /// diff --git a/tmva/tmva/src/DataSetInfo.cxx b/tmva/tmva/src/DataSetInfo.cxx index ce1ee67088138..82d6e13bc3c5e 100644 --- a/tmva/tmva/src/DataSetInfo.cxx +++ b/tmva/tmva/src/DataSetInfo.cxx @@ -85,10 +85,10 @@ TMVA::DataSetInfo::~DataSetInfo() ClearDataSet(); for(UInt_t i=0, iEnd = fClasses.size(); iclear(); delete fInputVars; } @@ -385,14 +385,14 @@ TMVA::MethodBase::~MethodBase( void ) if (fSplTrainRefB) { delete fSplTrainRefB; fSplTrainRefB = 0; } if (fSplTrainEffBvsS) { delete fSplTrainEffBvsS; fSplTrainEffBvsS = 0; } - for (Int_t i = 0; i < 2; i++ ) { + for (size_t i = 0; i < fEventCollections.size(); i++ ) { if (fEventCollections.at(i)) { for (std::vector::const_iterator it = fEventCollections.at(i)->begin(); it != fEventCollections.at(i)->end(); ++it) { delete (*it); } delete fEventCollections.at(i); - fEventCollections.at(i) = 0; + fEventCollections.at(i) = nullptr; } } @@ -878,7 +878,7 @@ void TMVA::MethodBase::AddClassifierOutput( Types::ETreeType type ) // use timer Timer timer( nEvents, GetName(), kTRUE ); - std::vector mvaValues = GetMvaValues(0, nEvents, true); + std::vector mvaValues(nEvents);// = GetMvaValues(0, nEvents, true); // store time used for testing if (type==Types::kTesting)