From 09da8a22c7b2f78c71edc450b130b8be1408c85a Mon Sep 17 00:00:00 2001 From: Jordan Sherer Date: Wed, 13 Nov 2019 16:10:26 -0500 Subject: [PATCH] Attempt decoder process recovery on hang until I determine why its hanging --- commons.h | 2 +- mainwindow.cpp | 164 ++++++++++++++++++++++++++++++++++++------------- mainwindow.h | 9 ++- 3 files changed, 128 insertions(+), 47 deletions(-) diff --git a/commons.h b/commons.h index 6659256..5a94694 100644 --- a/commons.h +++ b/commons.h @@ -10,7 +10,7 @@ #define JS8_USE_IHSYM 0 // compute ihsym manually instead of from symspec #define JS8_RING_BUFFER 1 // use a ring buffer instead of clearing the decode frames #define JS8_SINGLE_DECODE 0 // single submode decode per instantiation of the decoder -#define JS8_DEBUG_DECODE 0 // emit debug statements for the decode pipeline +#define JS8_DEBUG_DECODE 1 // emit debug statements for the decode pipeline #define JS8_NUM_SYMBOLS 79 #define JS8_ENABLE_JS8A 1 diff --git a/mainwindow.cpp b/mainwindow.cpp index 05d39b7..fbd237e 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -637,16 +637,6 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple, setWindowTitle (program_title ()); - connect(&proc_js8, &QProcess::readyReadStandardOutput, this, &MainWindow::readFromStdout); - connect(&proc_js8, static_cast (&QProcess::error), - [this] (QProcess::ProcessError error) { - subProcessError (&proc_js8, error); - }); - connect(&proc_js8, static_cast (&QProcess::finished), - [this] (int exitCode, QProcess::ExitStatus status) { - subProcessFailed (&proc_js8, exitCode, status); - }); - // hook up save WAV file exit handling connect (&m_saveWAVWatcher, &QFutureWatcher::finished, [this] { // extract the promise from the future @@ -853,34 +843,8 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple, } } - //Create .lock so jt9 will wait - QFile {m_config.temp_dir ().absoluteFilePath (".lock")}.open(QIODevice::ReadWrite); - - QStringList js8_args { - "-s", QApplication::applicationName () // shared memory key, - // includes rig -#ifdef NDEBUG - , "-w", "1" //FFTW patience - release -#else - , "-w", "1" //FFTW patience - debug builds for speed -#endif - // The number of threads for FFTW specified here is chosen as - // three because that gives the best throughput of the large - // FFTs used in jt9. The count is the minimum of (the number - // available CPU threads less one) and three. This ensures that - // there is always at least one free CPU thread to run the other - // mode decoder in parallel. - , "-m", QString::number (qMin (qMax (QThread::idealThreadCount () - 1, 1), 3)) //FFTW threads - - , "-e", QDir::toNativeSeparators (m_appDir) - , "-a", QDir::toNativeSeparators (m_config.writeable_data_dir ().absolutePath ()) - , "-t", QDir::toNativeSeparators (m_config.temp_dir ().absolutePath ()) - }; - QProcessEnvironment env {QProcessEnvironment::systemEnvironment ()}; - env.insert ("OMP_STACKSIZE", "4M"); - proc_js8.setProcessEnvironment (env); - proc_js8.start(QDir::toNativeSeparators (m_appDir) + QDir::separator () + - "js8", js8_args, QIODevice::ReadWrite | QIODevice::Unbuffered); + initDecoderSubprocess(); + decodeBusy(true); QString fname {QDir::toNativeSeparators(m_config.writeable_data_dir ().absoluteFilePath ("wsjtx_wisdom.dat"))}; QByteArray cfname=fname.toLocal8Bit(); @@ -1615,6 +1579,72 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple, if (!m_valid) throw std::runtime_error {"Fatal initialization exception"}; } +void MainWindow::initDecoderSubprocess(){ + //Create .lock so jt9 will wait + QFile {m_config.temp_dir ().absoluteFilePath (".lock")}.open(QIODevice::ReadWrite); + + QStringList js8_args { + "-s", QApplication::applicationName () // shared memory key, + // includes rig + #ifdef NDEBUG + , "-w", "1" //FFTW patience - release + #else + , "-w", "1" //FFTW patience - debug builds for speed + #endif + // The number of threads for FFTW specified here is chosen as + // three because that gives the best throughput of the large + // FFTs used in jt9. The count is the minimum of (the number + // available CPU threads less one) and three. This ensures that + // there is always at least one free CPU thread to run the other + // mode decoder in parallel. + , "-m", QString::number (qMin (qMax (QThread::idealThreadCount () - 1, 1), 3)) //FFTW threads + + , "-e", QDir::toNativeSeparators (m_appDir) + , "-a", QDir::toNativeSeparators (m_config.writeable_data_dir ().absolutePath ()) + , "-t", QDir::toNativeSeparators (m_config.temp_dir ().absolutePath ()) + }; + + QProcessEnvironment env {QProcessEnvironment::systemEnvironment ()}; + env.insert ("OMP_STACKSIZE", "4M"); + + if(JS8_DEBUG_DECODE) qDebug() << "decoder subprocess starting..."; + + auto proc = new QProcess(this); + proc->setProcessEnvironment (env); + proc->start(QDir::toNativeSeparators (m_appDir) + QDir::separator () + + "js8", js8_args, QIODevice::ReadWrite | QIODevice::Unbuffered); + + connect(proc, &QProcess::readyReadStandardOutput, this, + [this, proc](){ + readFromStdout(proc); + }); + + connect(proc, static_cast (&QProcess::error), + [this, proc] (QProcess::ProcessError error) { + subProcessError (proc, error); + }); + + connect(proc, static_cast (&QProcess::finished), + [this, proc] (int exitCode, QProcess::ExitStatus status) { + subProcessFailed (proc, exitCode, status); + }); + + // kill the previous proc and set the new one + m_valid = false; + { + if(!proc_js8.isNull()){ + proc_js8->kill(); + } + proc_js8.reset(proc); + } + m_valid = true; + + // reset decode busy + if(m_decoderBusy){ + decodeBusy(false); + } +} + QPair, int> splitVersion(QString v){ int hyphenPos = v.lastIndexOf("-"); if(hyphenPos >= 0){ @@ -3541,8 +3571,10 @@ void MainWindow::closeEvent(QCloseEvent * e) QFile quitFile {m_config.temp_dir ().absoluteFilePath (".quit")}; quitFile.open(QIODevice::ReadWrite); QFile {m_config.temp_dir ().absoluteFilePath (".lock")}.remove(); // Allow jt9 to terminate - bool b=proc_js8.waitForFinished(1000); - if(!b) proc_js8.close(); + if(!proc_js8.isNull()){ + bool b=proc_js8->waitForFinished(1000); + if(!b) proc_js8->close(); + } quitFile.remove(); Q_EMIT finished (); @@ -3993,6 +4025,8 @@ bool MainWindow::decode(){ if(JS8_DEBUG_DECODE) qDebug() << "decoder checking if ready..." << "k" << k << "k0" << kZero; + // TODO: check js8 process hasn't stalled? + if(isMessageQueuedForTransmit()){ if(JS8_DEBUG_DECODE) qDebug() << "--> decoder paused during transmit"; return false; @@ -4139,7 +4173,15 @@ bool MainWindow::decodeEnqueueReady(qint32 k, qint32 k0){ */ bool MainWindow::decodeProcessQueue(qint32 *pSubmode){ if(m_decoderBusy){ - if(JS8_DEBUG_DECODE) qDebug() << "--> decoder is busy!"; + int seconds = m_decoderBusyStartTime.secsTo(DriftingDateTime::currentDateTimeUtc()); + if(seconds > 60){ + if(JS8_DEBUG_DECODE) qDebug() << "--> decoder should be killed!" << QString("(%1 seconds)").arg(seconds); + } else if(seconds > 30){ + if(JS8_DEBUG_DECODE) qDebug() << "--> decoder is hanging!" << QString("(%1 seconds)").arg(seconds); + } else { + if(JS8_DEBUG_DECODE) qDebug() << "--> decoder is busy!"; + } + return false; } @@ -4377,6 +4419,7 @@ void MainWindow::decodeBusy(bool b) //decodeBusy() m_decoderBusy=b; if(m_decoderBusy){ tx_status_label.setText("Decoding"); + m_decoderBusyStartTime = DriftingDateTime::currentDateTimeUtc(); } ui->DecodeButton->setEnabled(!b); ui->actionOpen->setEnabled(!b); @@ -4432,6 +4475,33 @@ void MainWindow::decodePrepareSaveAudio(int submode){ } } +/** + * @brief MainWindow::decodeCheckHangingDecoder + * check if decoder is hanging and reset if it is + */ +void MainWindow::decodeCheckHangingDecoder(){ + if(!m_decoderBusy){ + return; + } + + if(m_decoderBusyStartTime.isValid() && m_decoderBusyStartTime.secsTo(DriftingDateTime::currentDateTimeUtc()) > 60){ + m_decoderBusyStartTime = QDateTime(); + + SelfDestructMessageBox * m = new SelfDestructMessageBox(60, + "Decoder Hang", + "The JS8 decoder is having trouble and is now restarting.", + QMessageBox::Warning, + QMessageBox::Ok, + QMessageBox::Ok, + false, + this); + + m->show(); + + initDecoderSubprocess(); + } +} + void MainWindow::writeAllTxt(QString message, int bits) { // Write decoded text to file "ALL.TXT". @@ -4530,11 +4600,16 @@ QList generateOffsets(int minOffset, int maxOffset){ return offsets; } -void MainWindow::readFromStdout() //readFromStdout +void MainWindow::readFromStdout(QProcess * proc) //readFromStdout { - while(proc_js8.canReadLine()) { - QByteArray t=proc_js8.readLine(); + if(!proc || proc->state() != QProcess::Running){ + return; + } + + while(proc->canReadLine()) { + QByteArray t = proc->readLine(); qDebug() << "JS8: " << QString(t); + bool bAvgMsg=false; int navg=0; if(t.indexOf("") >= 0) { @@ -5600,6 +5675,7 @@ void MainWindow::guiUpdate() // once per period if(m_sec0 % m_TRperiod == 0){ tryBandHop(); + decodeCheckHangingDecoder(); } // at the end of the period diff --git a/mainwindow.h b/mainwindow.h index a78aff9..0508129 100644 --- a/mainwindow.h +++ b/mainwindow.h @@ -114,6 +114,9 @@ public: QWidget *parent = nullptr); ~MainWindow(); +private: + void initDecoderSubprocess(); + public slots: void showSoundInError(const QString& errorMsg); void showSoundOutError(const QString& errorMsg); @@ -121,7 +124,7 @@ public slots: void dataSink(qint64 frames); void diskDat(); void guiUpdate(); - void readFromStdout(); + void readFromStdout(QProcess * proc); void setXIT(int n, Frequency base = 0u); void qsy(int hzDelta); void setFreqOffsetForRestore(int freq, bool shouldRestore); @@ -236,6 +239,7 @@ private slots: void decodePrepareSaveAudio(int submode); void decodeBusy(bool b); void decodeDone (); + void decodeCheckHangingDecoder(); void on_EraseButton_clicked(); void set_dateTimeQSO(int m_ntx); void set_ntx(int n); @@ -577,6 +581,7 @@ private: bool m_diskData; bool m_loopall; bool m_decoderBusy; + QDateTime m_decoderBusyStartTime; bool m_auto; bool m_restart; bool m_startAnother; @@ -660,7 +665,7 @@ private: QFutureWatcher watcher3; QFutureWatcher m_saveWAVWatcher; - QProcess proc_js8; + QScopedPointer proc_js8; QTimer m_guiTimer; QTimer ptt1Timer; //StartTx delay