From 440311a75b3300d3dace18bbb57d9badd493ac56 Mon Sep 17 00:00:00 2001 From: Jordan Sherer Date: Mon, 29 Oct 2018 03:26:10 -0400 Subject: [PATCH] Restructured data frame packing so we can send more over the wire in fewer frames --- decodedtext.cpp | 20 ++++++++---- decodedtext.h | 3 +- mainwindow.cpp | 51 ++++++++++++++++++------------ mainwindow.h | 9 +++--- varicode.cpp | 84 ++++++++++++++++++++++++++----------------------- varicode.h | 9 +++--- 6 files changed, 102 insertions(+), 74 deletions(-) diff --git a/decodedtext.cpp b/decodedtext.cpp index 3441415..18fdef5 100644 --- a/decodedtext.cpp +++ b/decodedtext.cpp @@ -24,6 +24,7 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString , frameType_(Varicode::FrameUnknown) , isHeartbeat_(false) , isAlt_(false) + , bits_{0} { if(message_.length() >= 1) { message_ = message_.left (21).remove (QRegularExpression {"[<>]"}); @@ -58,14 +59,17 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString } } + bits_ = bits(); + tryUnpack(); } -DecodedText::DecodedText (QString const& js8callmessage): +DecodedText::DecodedText (QString const& js8callmessage, int bits): frameType_(Varicode::FrameUnknown), message_(js8callmessage), isHeartbeat_(false), - isAlt_(false) + isAlt_(false), + bits_(bits) { is_standard_ = QRegularExpression("^(CQ|DE|QRZ)\\s").match(message_).hasMatch(); @@ -79,6 +83,10 @@ bool DecodedText::tryUnpack(){ } bool unpacked = false; + if(!unpacked){ + unpacked = tryUnpackData(); + } + if(!unpacked){ unpacked = tryUnpackHeartbeat(); } @@ -91,10 +99,6 @@ bool DecodedText::tryUnpack(){ unpacked = tryUnpackDirected(); } - if(!unpacked){ - unpacked = tryUnpackData(); - } - return unpacked; } @@ -212,6 +216,10 @@ bool DecodedText::tryUnpackData(){ return false; } + if((bits_ & Varicode::JS8CallData) != Varicode::JS8CallData){ + return false; + } + quint8 type = Varicode::FrameUnknown; QString data = Varicode::unpackDataMessage(m, &type); diff --git a/decodedtext.h b/decodedtext.h index 4a64a20..896eb2c 100644 --- a/decodedtext.h +++ b/decodedtext.h @@ -31,7 +31,7 @@ class DecodedText { public: explicit DecodedText (QString const& message, bool, QString const& my_grid); - explicit DecodedText (QString const& js8callmessage); + explicit DecodedText (QString const& js8callmessage, int bits); bool tryUnpack(); bool tryUnpackHeartbeat(); @@ -109,6 +109,7 @@ private: bool contest_mode_; QString message_; bool is_standard_; + int bits_; }; #endif // DECODEDTEXT_H diff --git a/mainwindow.cpp b/mainwindow.cpp index a91df7f..e4a47c2 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -2204,7 +2204,7 @@ void MainWindow::fastSink(qint64 frames) m_config.color_NewCall(),m_config.ppfx()); m_bDecoded=true; if (m_mode != "ISCAT") postDecode (true, decodedtext.string ()); - writeAllTxt(message); + writeAllTxt(message, decodedtext.bits()); bool stdMsg = decodedtext.report(m_baseCall, Radio::base_callsign(ui->dxCallEntry->text()),m_rptRcvd); //if (stdMsg) pskPost (decodedtext); @@ -3445,7 +3445,7 @@ void::MainWindow::fast_decode_done() m_bDecoded=true; } postDecode (true, decodedtext.string ()); - writeAllTxt(message); + writeAllTxt(message, decodedtext.bits()); if(m_mode=="JT9" or m_mode=="MSK144") { // find and extract any report for myCall @@ -3462,7 +3462,7 @@ void::MainWindow::fast_decode_done() m_bFastDone=false; } -void MainWindow::writeAllTxt(QString message) +void MainWindow::writeAllTxt(QString message, int bits) { // Write decoded text to file "ALL.TXT". QFile f {m_config.writeable_data_dir ().absoluteFilePath ("ALL.TXT")}; @@ -3474,7 +3474,7 @@ void MainWindow::writeAllTxt(QString message) << m_mode << endl; m_RxLog=0; } - auto dt = DecodedText(message); + auto dt = DecodedText(message, bits); out << dt.message() << endl; f.close(); } else { @@ -3569,7 +3569,7 @@ void MainWindow::readFromStdout() //readFromStdout (bits == Varicode::JS8Call || ((bits & Varicode::JS8CallFirst) == Varicode::JS8CallFirst) || ((bits & Varicode::JS8CallLast) == Varicode::JS8CallLast) || - ((bits & Varicode::JS8CallReserved) == 0 /*Varicode::JS8CallReserved*/)) // This is unused...so is invalid at this time... + ((bits & Varicode::JS8CallData) == Varicode::JS8CallData)) // This is unused...so is invalid at this time... ); qDebug() << "valid" << bValidFrame << "decoded text" << decodedtext.message(); @@ -4023,6 +4023,7 @@ void MainWindow::guiUpdate() static quint64 lastLoop; static char message[29]; static char msgsent[29]; + static int msgibits; double txDuration; QString rt; @@ -4170,11 +4171,12 @@ void MainWindow::guiUpdate() char ft8msgbits[75 + 12]; //packed 75 bit ft8 message plus 12-bit CRC genft8_(message, MyGrid, &bcontest, &m_i3bit, msgsent, const_cast (ft8msgbits), const_cast (itone), 22, 6, 22); - + msgibits = m_i3bit; msgsent[22]=0; } m_currentMessage = QString::fromLatin1(msgsent); + m_currentMessageBits = msgibits; m_bCallingCQ = CALLING == m_QSOProgress || m_currentMessage.contains (QRegularExpression {"^(CQ|QRZ) "}); if(m_mode=="FT8") { @@ -4322,7 +4324,7 @@ void MainWindow::guiUpdate() if(m_transmitting) { char s[41]; - auto dt = DecodedText(msgsent); + auto dt = DecodedText(msgsent, msgibits); sprintf(s,"Tx: %s", dt.message().toLocal8Bit().mid(0, 41).data()); m_nsendingsh=0; if(s[4]==64) m_nsendingsh=1; @@ -4486,7 +4488,7 @@ void MainWindow::startTx2() void MainWindow::stopTx() { Q_EMIT endTransmitMessage (); - auto dt = DecodedText(m_currentMessage.trimmed()); + auto dt = DecodedText(m_currentMessage.trimmed(), m_currentMessageBits); last_tx_label.setText("Last Tx: " + dt.message()); //m_currentMessage.trimmed()); m_btxok = false; @@ -5301,7 +5303,7 @@ void MainWindow::createMessageTransmitQueue(QString const& text){ QStringList lines; foreach(auto frame, frames){ - auto dt = DecodedText(frame); + auto dt = DecodedText(frame.first, frame.second); lines.append(dt.message()); } @@ -5324,9 +5326,9 @@ void MainWindow::resetMessageTransmitQueue(){ m_txMessageQueue.clear(); } -QString MainWindow::popMessageFrame(){ +QPair MainWindow::popMessageFrame(){ if(m_txFrameQueue.isEmpty()){ - return QString(); + return QPair{}; } return m_txFrameQueue.dequeue(); } @@ -5371,7 +5373,7 @@ int MainWindow::currentFreqOffset(){ return ui->RxFreqSpinBox->value(); } -QStringList MainWindow::buildMessageFrames(const QString &text){ +QList> MainWindow::buildMessageFrames(const QString &text){ // prepare selected callsign for directed message QString selectedCall = callsignSelected(); @@ -5395,7 +5397,7 @@ QStringList MainWindow::buildMessageFrames(const QString &text){ #if 0 qDebug() << "frames:"; foreach(auto frame, frames){ - auto dt = DecodedText(frame); + auto dt = DecodedText(frame.frame, frame.bits); qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType()); } #endif @@ -5407,7 +5409,10 @@ bool MainWindow::prepareNextMessageFrame() { m_i3bit = Varicode::JS8Call; - QString frame = popMessageFrame(); + QPair f = popMessageFrame(); + auto frame = f.first; + auto bits = f.second; + if(frame.isEmpty()){ ui->nextFreeTextMsg->clear(); updateTxButtonDisplay(); @@ -5416,6 +5421,7 @@ bool MainWindow::prepareNextMessageFrame() } else { ui->nextFreeTextMsg->setText(frame); + /* int count = m_txFrameCount; int sent = count - m_txFrameQueue.count(); @@ -5425,6 +5431,9 @@ bool MainWindow::prepareNextMessageFrame() if(count == sent){ m_i3bit |= Varicode::JS8CallLast; } + */ + + m_i3bit = bits; updateTxButtonDisplay(); @@ -8011,8 +8020,8 @@ void MainWindow::refreshTextDisplay(){ QStringList textList; qDebug() << "frames:"; - foreach(auto frame, frames){ - auto dt = DecodedText(frame); + foreach(Frame frame, frames){ + auto dt = DecodedText(frame.frame, frame.bits); qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType()); textList.append(dt.message()); } @@ -8053,14 +8062,16 @@ void MainWindow::refreshTextDisplay(){ ); connect(t, &BuildMessageFramesThread::finished, t, &QObject::deleteLater); - connect(t, &BuildMessageFramesThread::resultReady, this, [this, text](const QStringList frames){ + connect(t, &BuildMessageFramesThread::resultReady, this, [this, text](QStringList frames, QList bits){ QStringList textList; qDebug() << "frames:"; + int i = 0; foreach(auto frame, frames){ - auto dt = DecodedText(frame); + auto dt = DecodedText(frame, bits.at(i)); qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType()); textList.append(dt.message()); + i++; } auto transmitText = textList.join(""); @@ -8389,7 +8400,7 @@ void MainWindow::processCompoundActivity() { bits == Varicode::JS8Call || ((bits & Varicode::JS8CallFirst) == Varicode::JS8CallFirst) || ((bits & Varicode::JS8CallLast) == Varicode::JS8CallLast) || - ((bits & Varicode::JS8CallReserved) == Varicode::JS8CallReserved) + ((bits & Varicode::JS8CallData) == Varicode::JS8CallData) ); if (!validBits) { qDebug() << "-> buffer.cmd bits is invalid...skip"; @@ -10346,7 +10357,7 @@ void MainWindow::write_transmit_entry (QString const& file_name) QTextStream out(&f); auto time = DriftingDateTime::currentDateTimeUtc (); time = time.addSecs (-(time.time ().second () % m_TRperiod)); - auto dt = DecodedText(m_currentMessage); + auto dt = DecodedText(m_currentMessage, m_currentMessageBits); out << time.toString("yyyy-MM-dd hh:mm:ss") << " Transmitting " << qSetRealNumberPrecision (12) << (m_freqNominal / 1.e6) << " MHz " << QString(m_modeTx).replace("FT8", "JS8") diff --git a/mainwindow.h b/mainwindow.h index 129cf1a..4789c6e 100644 --- a/mainwindow.h +++ b/mainwindow.h @@ -155,7 +155,7 @@ public slots: void createMessage(QString const& text); void createMessageTransmitQueue(QString const& text); void resetMessageTransmitQueue(); - QString popMessageFrame(); + QPair popMessageFrame(); protected: void keyPressEvent (QKeyEvent *) override; void closeEvent(QCloseEvent *) override; @@ -298,7 +298,7 @@ private slots: void on_nextFreeTextMsg_currentTextChanged (QString const&); void on_extFreeTextMsgEdit_currentTextChanged (QString const&); int currentFreqOffset(); - QStringList buildMessageFrames(QString const& text); + QList> buildMessageFrames(QString const& text); bool prepareNextMessageFrame(); bool isFreqOffsetFree(int f, int bw); int findFreeFreqOffset(int fmin, int fmax, int bw); @@ -415,7 +415,7 @@ private: private: void astroUpdate (); - void writeAllTxt(QString message); + void writeAllTxt(QString message, int bits); void hideMenus(bool b); NetworkAccessManager m_network_manager; @@ -553,6 +553,7 @@ private: bool m_sentFirst73; int m_currentMessageType; QString m_currentMessage; + int m_currentMessageBits; int m_lastMessageType; QString m_lastMessageSent; bool m_bShMsgs; @@ -773,7 +774,7 @@ private: QMap m_showColumnsCache; // table column:key -> show boolean QMap m_sortCache; // table key -> sort by QPriorityQueue m_txMessageQueue; // messages to be sent - QQueue m_txFrameQueue; // frames to be sent + QQueue> m_txFrameQueue; // frames to be sent QQueue m_rxActivityQueue; // all rx activity queue QQueue m_rxCommandQueue; // command queue for processing commands QQueue m_rxCallQueue; // call detail queue for spots to pskreporter diff --git a/varicode.cpp b/varicode.cpp index 9d89e7d..43d0d55 100644 --- a/varicode.cpp +++ b/varicode.cpp @@ -104,7 +104,7 @@ QString callsign_pattern = QString("(?[@]?[A-Z0-9/]+)"); QString optional_cmd_pattern = QString("(?\\s?(?:HEARTBEAT (ACK|REQ)|AGN[?]|QSL[?]|HW CPY[?]|APRS[:]|QRZ[?]|SNR[?]|QTC[?]|QTH[?]|GRID[?]|STATUS[?]|(?:(?:ACK|73|YES|NO|SNR|QSL|RR|SK|FB|QTH|QTC|GRID|ACTIVE|IDLE)(?=[ ]|$))|[#> ]))?"); QString optional_grid_pattern = QString("(?\\s?[A-R]{2}[0-9]{2})?"); QString optional_extended_grid_pattern = QString("^(?\\s?(?:[A-R]{2}[0-9]{2}(?:[A-X]{2}(?:[0-9]{2})?)*))?"); -QString optional_num_pattern = QString("(?(?<=SNR|ACK)\\s?[-+]?(?:3[01]|[0-2]?[0-9]))?"); +QString optional_num_pattern = QString("(?(?<=SNR|HEARTBEAT ACK)\\s?[-+]?(?:3[01]|[0-2]?[0-9]))?"); QRegularExpression directed_re("^" + callsign_pattern + @@ -981,7 +981,7 @@ quint8 Varicode::packCmd(quint8 cmd, quint8 num, bool *pPackedNum){ // [1][X][6] // X = 0 == SNR // X = 1 == ACK - value = ((1 << 1) | (int)(cmdStr == " ACK")) << 6; + value = ((1 << 1) | (int)(cmdStr == " HEARTBEAT ACK")) << 6; value = value + (num & ((1<<6)-1)); if(pPackedNum) *pPackedNum = true; } else { @@ -999,7 +999,7 @@ quint8 Varicode::unpackCmd(quint8 value, quint8 *pNum){ auto cmd = directed_cmds[" SNR"]; if(value & (1<<6)){ - cmd = directed_cmds[" ACK"]; + cmd = directed_cmds[" HEARTBEAT ACK"]; } return cmd; } else { @@ -1459,12 +1459,10 @@ QStringList Varicode::unpackDirectedMessage(const QString &text, quint8 *pType){ QString packHuffMessage(const QString &input, int *n){ static const int frameSize = 72; - QString frame; + QString frame = {false}; - // [3][69] = 72 - QVector frameDataBits; - - QVector frameHeaderBits = Varicode::intToBits(Varicode::FrameDataUncompressed, 3); + // [1][71] = 72 + QVector frameBits = {false}; int i = 0; @@ -1474,6 +1472,7 @@ QString packHuffMessage(const QString &input, int *n){ for(it = input.constBegin(); it != input.constEnd(); it++){ auto ch = (*it).toUpper(); if(!validChars.contains(ch)){ + if(n) *n = 0; return frame; } } @@ -1482,32 +1481,28 @@ QString packHuffMessage(const QString &input, int *n){ foreach(auto pair, Varicode::huffEncode(Varicode::defaultHuffTable(), input)){ auto charN = pair.first; auto charBits = pair.second; - if(frameHeaderBits.length() + frameDataBits.length() + charBits.length() < frameSize){ - frameDataBits += charBits; + if(frameBits.length() + charBits.length() < frameSize){ + frameBits += charBits; i += charN; continue; } break; } - QVector framePadBits; + qDebug() << "Huff bits" << frameBits.length() << "chars" << i; - int pad = frameSize - frameHeaderBits.length() - frameDataBits.length(); + int pad = frameSize - frameBits.length(); if(pad){ // the way we will pad is this... // set the bit after the frame to 0 and every bit after that a 1 // to unpad, seek from the end of the bits until you hit a zero... the rest is the actual frame. for(int i = 0; i < pad; i++){ - framePadBits.append(i == 0 ? (bool)0 : (bool)1); + frameBits.append(i == 0 ? (bool)0 : (bool)1); } } - qDebug() << "Huff bits" << frameDataBits.length() << "chars" << i; - - QVector allBits = frameHeaderBits + frameDataBits + framePadBits; - - quint64 value = Varicode::bitsToInt(allBits.constBegin(), 64); - quint8 rem = (quint8)Varicode::bitsToInt(allBits.constBegin() + 64, 8); + quint64 value = Varicode::bitsToInt(frameBits.constBegin(), 64); + quint8 rem = (quint8)Varicode::bitsToInt(frameBits.constBegin() + 64, 8); frame = Varicode::pack72bits(value, rem); if(n) *n = i; @@ -1520,9 +1515,8 @@ QString packCompressedMessage(const QString &input, int *n){ QString frame; - QVector frameBits; - - frameBits.append(Varicode::intToBits(Varicode::FrameDataCompressed, 3)); + // [1][71] = 72 + QVector frameBits = {true}; int i = 0; foreach(auto pair, JSC::compress(input)){ @@ -1538,7 +1532,7 @@ QString packCompressedMessage(const QString &input, int *n){ break; } - qDebug() << "Compressed bits" << frameBits.length() - 3 << "chars" << i; + qDebug() << "Compressed bits" << frameBits.length() << "chars" << i; int pad = frameSize - frameBits.length(); if(pad){ @@ -1588,25 +1582,25 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){ quint64 value = Varicode::unpack72bits(text, &rem); auto bits = Varicode::intToBits(value, 64) + Varicode::intToBits(rem, 8); - quint8 type = Varicode::bitsToInt(bits.mid(0, 3)); + bool compressed = bits.at(0); int n = bits.lastIndexOf(0); - bits = bits.mid(3, n-3); + bits = bits.mid(1, n-1); - if(type == FrameDataUncompressed){ + if(compressed){ + unpacked = JSC::decompress(bits); + if(pType) *pType = Varicode::FrameDataCompressed; + } else { // huff decode the bits (without escapes) unpacked = Varicode::huffDecode(Varicode::defaultHuffTable(), bits); - if(pType) *pType = type; - } else if(type == FrameDataCompressed) { - unpacked = JSC::decompress(bits); - if(pType) *pType = type; + if(pType) *pType = Varicode::FrameDataUncompressed; } return unpacked; } // TODO: remove the dependence on providing all this data? -QStringList Varicode::buildMessageFrames( +QList> Varicode::buildMessageFrames( QString const& mycall, //QString const& basecall, QString const& mygrid, @@ -1621,7 +1615,7 @@ QStringList Varicode::buildMessageFrames( bool mycallCompound = Varicode::isCompoundCallsign(mycall); - QStringList frames; + QList> frames; foreach(QString line, text.split(QRegExp("[\\r\\n]"), QString::SkipEmptyParts)){ // once we find a directed call, data encode the rest of the line. @@ -1735,13 +1729,13 @@ QStringList Varicode::buildMessageFrames( } if(useBcn){ - frames.append(frame); + frames.append({ frame, Varicode::JS8Call }); line = line.mid(l); } #if ALLOW_SEND_COMPOUND if(useCmp){ - frames.append(frame); + frames.append({ frame, Varicode::JS8Call }); line = line.mid(o); } #endif @@ -1775,14 +1769,14 @@ QStringList Varicode::buildMessageFrames( QString deCompoundMessage = QString("`%1 %2").arg(mycall).arg(mygrid); QString deCompoundFrame = Varicode::packCompoundMessage(deCompoundMessage, nullptr); if(!deCompoundFrame.isEmpty()){ - frames.append(deCompoundFrame); + frames.append({ deCompoundFrame, Varicode::JS8Call }); } // Followed, by a standard OR compound directed message... QString dirCompoundMessage = QString("`%1%2%3").arg(dirTo).arg(dirCmd).arg(dirNum); QString dirCompoundFrame = Varicode::packCompoundMessage(dirCompoundMessage, nullptr); if(!dirCompoundFrame.isEmpty()){ - frames.append(dirCompoundFrame); + frames.append({ dirCompoundFrame, Varicode::JS8Call }); } shouldUseStandardFrame = false; } @@ -1790,7 +1784,7 @@ QStringList Varicode::buildMessageFrames( if(shouldUseStandardFrame) { // otherwise, just send the standard directed frame - frames.append(frame); + frames.append({ frame, Varicode::JS8Call }); } line = line.mid(n); @@ -1818,12 +1812,17 @@ QStringList Varicode::buildMessageFrames( } if(useDat){ - frames.append(frame); + frames.append({ frame, Varicode::JS8CallData }); line = line.mid(m); } } } + if(!frames.isEmpty()){ + frames.first().second |= Varicode::JS8CallFirst; + frames.last().second |= Varicode::JS8CallLast; + } + return frames; } @@ -1854,5 +1853,12 @@ void BuildMessageFramesThread::run(){ m_text ); - emit resultReady(results); + QList frames; + QList bits; + foreach(auto pair, results){ + frames.append(pair.first); + bits.append(pair.second); + } + + emit resultReady(frames, bits); } diff --git a/varicode.h b/varicode.h index 0cf4fed..994f6d7 100644 --- a/varicode.h +++ b/varicode.h @@ -12,6 +12,7 @@ #include #include + class Varicode { public: @@ -20,12 +21,12 @@ public: JS8Call = 0, // [000] <- any other frame of the message JS8CallFirst = 1, // [001] <- the first frame of a message JS8CallLast = 2, // [010] <- the last frame of a message - JS8CallReserved = 4, // [100] <- a reserved flag for future use... + JS8CallData = 4, // [100] <- raw data frame (no frame type header) }; enum FrameType { FrameUnknown = 255, // [11111111] <- only used as a sentinel - FrameHeartbeat = 0, // [000] + FrameHeartbeat = 0, // [000] FrameCompound = 1, // [001] FrameCompoundDirected = 2, // [010] FrameDirected = 3, // [011] @@ -147,7 +148,7 @@ public: static QString packDataMessage(QString const& text, int *n); static QString unpackDataMessage(QString const& text, quint8 *pType); - static QStringList buildMessageFrames( + static QList> buildMessageFrames( QString const& mycall, //QString const& basecall, QString const& mygrid, @@ -171,7 +172,7 @@ public: QObject *parent=nullptr); void run() override; signals: - void resultReady(const QStringList s); + void resultReady(QStringList, QList); private: QString m_mycall;