From b8267372e466c22cac903281fe406a995bbd01e5 Mon Sep 17 00:00:00 2001 From: Jordan Sherer Date: Thu, 19 Jul 2018 03:44:02 -0400 Subject: [PATCH] Added varicode encoding of messages What this does is allow us to pack more than 13 characters in a single transmission frame. Optimized using a Huffman encoding using weights of alphabetical frequency, this will often allow us to send less than 5 bits per character. --- decodedtext.cpp | 65 ++++++++++++++++++++++++++++++------------- decodedtext.h | 3 +- mainwindow.cpp | 38 ++++++++++++++++++------- varicode.cpp | 74 +++++++++++++++++++++++++++++++++++++++++++------ varicode.h | 5 +++- 5 files changed, 146 insertions(+), 39 deletions(-) diff --git a/decodedtext.cpp b/decodedtext.cpp index 236faa6..b076cd5 100644 --- a/decodedtext.cpp +++ b/decodedtext.cpp @@ -55,33 +55,60 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString } if(!is_standard_){ - tryUnpackDirected(); + bool unpacked = false; + + if(!unpacked){ + unpacked = tryUnpackDirected(); + } + if(!unpacked){ + unpacked = tryUnpackData(); + } + } } -void DecodedText::tryUnpackDirected(){ - QString m = message().trimmed(); +bool DecodedText::tryUnpackDirected(){ + QString m = message().trimmed(); - // directed calls will always be 12+ chars and contain no spaces. - if(m.length() < 12 || m.contains(' ')){ - return; - } + // directed calls will always be 12+ chars and contain no spaces. + if(m.length() < 12 || m.contains(' ')){ + return false; + } - QStringList parts = Varicode::unpackDirectedMessage(m); + QStringList parts = Varicode::unpackDirectedMessage(m); - if(parts.isEmpty()){ - return; - } + if(parts.isEmpty()){ + return false; + } - if(parts.length() == 3){ - // replace it with the correct unpacked (query) - message_ = QString("%1: %2%3").arg(parts.at(0), parts.at(1), parts.at(2)); - } else { - // replace it with the correct unpacked (freetext) - message_ = QString(parts.join(QChar())); - } + if(parts.length() == 3){ + // replace it with the correct unpacked (query) + message_ = QString("%1: %2%3").arg(parts.at(0), parts.at(1), parts.at(2)); + } else { + // replace it with the correct unpacked (freetext) + message_ = QString(parts.join(QChar())); + } - directed_ = parts; + directed_ = parts; + return true; +} + +bool DecodedText::tryUnpackData(){ + QString m = message().trimmed(); + + // data frames calls will always be 12+ chars and contain no spaces. + if(m.length() < 12 || m.contains(' ')){ + return false; + } + + QString data = Varicode::unpackDataMessage(m); + + if(data.isEmpty()){ + return false; + } + + message_ = data; + return true; } QStringList DecodedText::messageWords () const diff --git a/decodedtext.h b/decodedtext.h index f41aaec..bd954e4 100644 --- a/decodedtext.h +++ b/decodedtext.h @@ -32,7 +32,8 @@ class DecodedText public: explicit DecodedText (QString const& message, bool, QString const& my_grid); - void tryUnpackDirected(); + bool tryUnpackDirected(); + bool tryUnpackData(); QStringList directedMessage() const { return directed_; } bool isDirectedMessage() const { return !directed_.isEmpty(); } diff --git a/mainwindow.cpp b/mainwindow.cpp index f6c6804..9a96b0e 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -3337,6 +3337,7 @@ void MainWindow::readFromStdout() //readFromStdout d.bits = decodedtext.bits(); d.freq = audioFreq; d.text = decodedtext.message(); + qDebug() << d.text; d.utcTimestamp = QDateTime::currentDateTimeUtc(); m_rxFrameQueue.append(d); } @@ -5297,10 +5298,8 @@ int MainWindow::logRxTxMessageText(QDateTime date, bool isFree, QString text, in } if(found){ - if(!isFree){ - c.insertText(" "); - } - c.insertHtml(text); + c.clearSelection(); + c.insertText(text); } else { c.insertHtml(QString("%1 - (%2) - %3").arg(date.time().toString()).arg(freq).arg(text)); } @@ -5458,22 +5457,33 @@ QPair MainWindow::buildFT8MessageFrames(QString const& QString mycall = m_config.my_callsign(); foreach(QString line, text.split(QRegExp("[\\r\\n]"), QString::SkipEmptyParts)){ + + + while(line.size() > 0){ QString frame; bool useStd = false; + bool useDir = false; + bool useDat = false; bool isFree = false; QString stdFrame = parseFT8Message(line, &isFree); int n = 0; QString dirFrame = Varicode::packDirectedMessage(line, mycall, &n); + int m = 0; + QString datFrame = Varicode::packDataMessage(line.left(21) + "\x04", &m); // 63 / 3 = 21 (maximum number of 3bit chars we could possibly stuff in here) + // if this parses to a standard FT8 free text message // but it can be parsed as a directed message, then we // should send the directed version if(isFree && n > 0){ - useStd = false; + useDir = true; frame = dirFrame; + } else if (isFree && m > 0) { + useDat = true; + frame = datFrame; } else { useStd = true; frame = stdFrame; @@ -5494,7 +5504,9 @@ QPair MainWindow::buildFT8MessageFrames(QString const& } line = line.mid(frame.length()).trimmed(); - } else { + } + + if(useDir){ frames.append(frame); // TODO: jsherer - would be nice to clean this up and have an object that can just decode the actual transmitted frames instead. if(!line.startsWith(mycall)){ @@ -5503,18 +5515,24 @@ QPair MainWindow::buildFT8MessageFrames(QString const& lines.append(line.left(n)); line = line.mid(n).trimmed(); } + + if(useDat){ + frames.append(frame); + lines.append(line.left(m)); + line = line.mid(m); + } } } -#if 0 +#if 1 qDebug() << "parsed frames:"; foreach(auto frame, frames){ - qDebug() << "->" << frame; + qDebug() << "->" << frame << Varicode::unpackDataMessage(frame); } qDebug() << "lines:"; - foreach(auto frame, frames){ - qDebug() << "->" << frame; + foreach(auto line, lines){ + qDebug() << "->" << line; } #endif diff --git a/varicode.cpp b/varicode.cpp index 486266c..69b26bd 100644 --- a/varicode.cpp +++ b/varicode.cpp @@ -175,10 +175,10 @@ QVector Varicode::huffFlatten(QList> &list){ return out; } -QString Varicode::huffDecode(QVector const& bitvec){ +QString Varicode::huffDecode(QVector const& bitvec, int pad){ QString out; - QString bits = bitsToStr(bitvec); + QString bits = bitsToStr(bitvec).mid(0, bitvec.length()-pad); // TODO: jsherer - this is naive... while(bits.length() > 0){ @@ -538,6 +538,7 @@ QString Varicode::packDirectedMessage(const QString &text, const QString &callsi auto fromBytes = from.toLocal8Bit(); auto fromCRC = CRC::Calculate(fromBytes.data(), fromBytes.length(), CRC::CRC_5_ITU()); + quint8 packed_is_data = 0; quint8 packed_flag = 0; quint32 packed_from = Varicode::packCallsign(from); quint32 packed_to = Varicode::packCallsign(to); @@ -550,18 +551,18 @@ QString Varicode::packDirectedMessage(const QString &text, const QString &callsi quint8 packed_cmd = directed_cmds[cmd]; quint8 packed_extra = fromCRC; - // [3][28][28][5],[5] = 69 + // [1][2][28][28][5],[5] = 69 auto bits = ( - Varicode::intToBits(packed_flag, 3) + + Varicode::intToBits(packed_is_data, 1) + + Varicode::intToBits(packed_flag, 2) + Varicode::intToBits(packed_from, 28) + Varicode::intToBits(packed_to, 28) + Varicode::intToBits(packed_cmd & 31, 5) ); frame = Varicode::pack64bits(Varicode::bitsToInt(bits)) + Varicode::pack5bits(packed_extra & 31); *n = match.captured(0).length(); + return frame; } - - return frame; } QStringList Varicode::unpackDirectedMessage(const QString &text){ @@ -571,11 +572,15 @@ QStringList Varicode::unpackDirectedMessage(const QString &text){ return unpacked; } - // [3][28][28][5],[5] = 69 + // [1][2][28][28][5],[5] = 69 auto bits = Varicode::bitsToStr(Varicode::intToBits(Varicode::unpack64bits(text.left(12)), 64)); quint8 extra = Varicode::unpack5bits(text.right(1)); - quint8 flag = Varicode::bitsToInt(Varicode::strToBits(bits.left(3))); + quint8 is_data = Varicode::bitsToInt(Varicode::strToBits(bits.left(1))); + if(is_data != 0){ + return unpacked; + } + quint8 flag = Varicode::bitsToInt(Varicode::strToBits(bits.mid(1,2))); quint32 packed_from = Varicode::bitsToInt(Varicode::strToBits(bits.mid(3, 28))); quint32 packed_to = Varicode::bitsToInt(Varicode::strToBits(bits.mid(31, 28))); quint8 packed_cmd = Varicode::bitsToInt(Varicode::strToBits(bits.mid(59, 5))); @@ -594,3 +599,56 @@ QStringList Varicode::unpackDirectedMessage(const QString &text){ return unpacked; } + +QString Varicode::packDataMessage(const QString &text, int *n){ + QString frame; + + // [1][63],[5] + quint8 is_data = 1; + auto frameBits = ( + Varicode::intToBits(is_data, 1) + ); + + int i = 0; + foreach(auto charBits, Varicode::huffEncode(text)){ + if(frameBits.length() + charBits.length() < 63){ + frameBits += charBits; + i++; + continue; + } + break; + } + + int pad = 64 - frameBits.length(); + if(pad){ + frameBits += Varicode::intToBits(1, pad); + } + + frame = Varicode::pack64bits(Varicode::bitsToInt(frameBits)) + Varicode::pack5bits(pad & 31); + *n = i; + + return frame; +} + +QString Varicode::unpackDataMessage(const QString &text){ + QString unpacked; + + if(text.length() < 13){ + return unpacked; + } + + auto bits = Varicode::intToBits(Varicode::unpack64bits(text.left(12)), 64); + quint8 pad = Varicode::unpack5bits(text.right(1)); + + quint8 is_data = (int)bits.at(0); + if(is_data != 1){ + return unpacked; + } + + // pop off the is_data bit + bits.removeAt(0); + + unpacked = Varicode::huffDecode(bits, pad); + + return unpacked; +} diff --git a/varicode.h b/varicode.h index 7756bb7..4299b6b 100644 --- a/varicode.h +++ b/varicode.h @@ -32,7 +32,7 @@ public: static QList> huffEncode(QString const& text); static QVector huffFlatten(QList> &list); - static QString huffDecode(QVector const& bitvec); + static QString huffDecode(QVector const& bitvec, int pad=0); static QVector bytesToBits(char * bitvec, int n); static QVector strToBits(QString const& bitvec); @@ -63,6 +63,9 @@ public: static bool isCommandAllowed(const QString &cmd); static QString packDirectedMessage(QString const& text, QString const& callsign, int *n); static QStringList unpackDirectedMessage(QString const& text); + + static QString packDataMessage(QString const& text, int *n); + static QString unpackDataMessage(QString const& text); }; #endif // VARICODE_H