Added varicode encoding of messages

What this does is allow us to pack more than 13 characters in a single
transmission frame. Optimized using a Huffman encoding using weights of
alphabetical frequency, this will often allow us to send less than 5 bits
per character.

decodedtext.cpp

@@ -55,22 +55,30 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString
     }
 
     if(!is_standard_){
-      tryUnpackDirected();
+      bool unpacked = false;
+
+      if(!unpacked){
+        unpacked = tryUnpackDirected();
+      }
+      if(!unpacked){
+        unpacked = tryUnpackData();
+      }
+
     }
 }
 
-void DecodedText::tryUnpackDirected(){
+bool DecodedText::tryUnpackDirected(){
     QString m = message().trimmed();
 
     // directed calls will always be 12+ chars and contain no spaces.
     if(m.length() < 12 || m.contains(' ')){
-    return;
+      return false;
     }
 
     QStringList parts = Varicode::unpackDirectedMessage(m);
 
     if(parts.isEmpty()){
-    return;
+      return false;
     }
 
     if(parts.length() == 3){
@@ -82,6 +90,25 @@ void DecodedText::tryUnpackDirected(){
     }
 
     directed_ = parts;
+    return true;
+}
+
+bool DecodedText::tryUnpackData(){
+    QString m = message().trimmed();
+
+    // data frames calls will always be 12+ chars and contain no spaces.
+    if(m.length() < 12 || m.contains(' ')){
+      return false;
+    }
+
+    QString data = Varicode::unpackDataMessage(m);
+
+    if(data.isEmpty()){
+      return false;
+    }
+
+    message_ = data;
+    return true;
 }
 
 QStringList DecodedText::messageWords () const

decodedtext.h

@@ -32,7 +32,8 @@ class DecodedText
 public:
   explicit DecodedText (QString const& message, bool, QString const& my_grid);
 
-  void tryUnpackDirected();
+  bool tryUnpackDirected();
+  bool tryUnpackData();
 
   QStringList directedMessage() const { return directed_; }
   bool isDirectedMessage() const { return !directed_.isEmpty(); }

mainwindow.cpp

@@ -3337,6 +3337,7 @@ void MainWindow::readFromStdout()                             //readFromStdout
         d.bits = decodedtext.bits();
         d.freq = audioFreq;
         d.text = decodedtext.message();
+        qDebug() << d.text;
         d.utcTimestamp = QDateTime::currentDateTimeUtc();
         m_rxFrameQueue.append(d);
       }
@@ -5297,10 +5298,8 @@ int MainWindow::logRxTxMessageText(QDateTime date, bool isFree, QString text, in
     }
 
     if(found){
-        if(!isFree){
+        c.clearSelection();
-            c.insertText(" ");
+        c.insertText(text);
-        }
-        c.insertHtml(text);
     } else {
         c.insertHtml(QString("<strong>%1 - (%2)</strong> - %3").arg(date.time().toString()).arg(freq).arg(text));
     }
@@ -5458,22 +5457,33 @@ QPair<QStringList, QStringList> MainWindow::buildFT8MessageFrames(QString const&
 
     QString mycall = m_config.my_callsign();
     foreach(QString line, text.split(QRegExp("[\\r\\n]"), QString::SkipEmptyParts)){
+
+
+
         while(line.size() > 0){
           QString frame;
 
           bool useStd = false;
+          bool useDir = false;
+          bool useDat = false;
           bool isFree = false;
           QString stdFrame = parseFT8Message(line, &isFree);
 
           int n = 0;
           QString dirFrame = Varicode::packDirectedMessage(line, mycall, &n);
 
+          int m = 0;
+          QString datFrame = Varicode::packDataMessage(line.left(21) + "\x04", &m); //  63 / 3 = 21 (maximum number of 3bit chars we could possibly stuff in here)
+
           // if this parses to a standard FT8 free text message
           // but it can be parsed as a directed message, then we
           // should send the directed version
           if(isFree && n > 0){
-              useStd = false;
+              useDir = true;
               frame = dirFrame;
+          } else if (isFree && m > 0) {
+              useDat = true;
+              frame = datFrame;
           } else {
               useStd = true;
               frame = stdFrame;
@@ -5494,7 +5504,9 @@ QPair<QStringList, QStringList> MainWindow::buildFT8MessageFrames(QString const&
               }
 
               line = line.mid(frame.length()).trimmed();
-          } else {
+          }
+
+          if(useDir){
               frames.append(frame);
               // TODO: jsherer - would be nice to clean this up and have an object that can just decode the actual transmitted frames instead.
               if(!line.startsWith(mycall)){
@@ -5503,18 +5515,24 @@ QPair<QStringList, QStringList> MainWindow::buildFT8MessageFrames(QString const&
               lines.append(line.left(n));
               line = line.mid(n).trimmed();
           }
+
+          if(useDat){
+              frames.append(frame);
+              lines.append(line.left(m));
+              line = line.mid(m);
+          }
         }
     }
 
-#if 0
+#if 1
     qDebug() << "parsed frames:";
     foreach(auto frame, frames){
-        qDebug() << "->" << frame;
+        qDebug() << "->" << frame << Varicode::unpackDataMessage(frame);
     }
 
     qDebug() << "lines:";
-    foreach(auto frame, frames){
+    foreach(auto line, lines){
-        qDebug() << "->" << frame;
+        qDebug() << "->" << line;
     }
 #endif
 

varicode.cpp

@@ -175,10 +175,10 @@ QVector<bool> Varicode::huffFlatten(QList<QVector<bool>> &list){
     return out;
 }
 
-QString Varicode::huffDecode(QVector<bool> const& bitvec){
+QString Varicode::huffDecode(QVector<bool> const& bitvec, int pad){
     QString out;
 
-    QString bits = bitsToStr(bitvec);
+    QString bits = bitsToStr(bitvec).mid(0, bitvec.length()-pad);
 
     // TODO: jsherer - this is naive...
     while(bits.length() > 0){
@@ -538,6 +538,7 @@ QString Varicode::packDirectedMessage(const QString &text, const QString &callsi
         auto fromBytes = from.toLocal8Bit();
         auto fromCRC = CRC::Calculate(fromBytes.data(), fromBytes.length(), CRC::CRC_5_ITU());
 
+        quint8 packed_is_data = 0;
         quint8 packed_flag = 0;
         quint32 packed_from = Varicode::packCallsign(from);
         quint32 packed_to = Varicode::packCallsign(to);
@@ -550,19 +551,19 @@ QString Varicode::packDirectedMessage(const QString &text, const QString &callsi
         quint8 packed_cmd = directed_cmds[cmd];
         quint8 packed_extra = fromCRC;
 
-        // [3][28][28][5],[5] = 69
+        // [1][2][28][28][5],[5] = 69
         auto bits = (
-            Varicode::intToBits(packed_flag, 3)    +
+            Varicode::intToBits(packed_is_data, 1) +
+            Varicode::intToBits(packed_flag, 2)    +
             Varicode::intToBits(packed_from, 28)   +
             Varicode::intToBits(packed_to, 28)     +
             Varicode::intToBits(packed_cmd & 31, 5)
         );
         frame = Varicode::pack64bits(Varicode::bitsToInt(bits)) + Varicode::pack5bits(packed_extra & 31);
         *n = match.captured(0).length();
-    }
-
         return frame;
     }
+}
 
 QStringList Varicode::unpackDirectedMessage(const QString &text){
     QStringList unpacked;
@@ -571,11 +572,15 @@ QStringList Varicode::unpackDirectedMessage(const QString &text){
         return unpacked;
     }
 
-    // [3][28][28][5],[5] = 69
+    // [1][2][28][28][5],[5] = 69
     auto bits = Varicode::bitsToStr(Varicode::intToBits(Varicode::unpack64bits(text.left(12)), 64));
     quint8 extra = Varicode::unpack5bits(text.right(1));
 
-    quint8 flag = Varicode::bitsToInt(Varicode::strToBits(bits.left(3)));
+    quint8 is_data = Varicode::bitsToInt(Varicode::strToBits(bits.left(1)));
+    if(is_data != 0){
+        return unpacked;
+    }
+    quint8 flag = Varicode::bitsToInt(Varicode::strToBits(bits.mid(1,2)));
     quint32 packed_from = Varicode::bitsToInt(Varicode::strToBits(bits.mid(3, 28)));
     quint32 packed_to = Varicode::bitsToInt(Varicode::strToBits(bits.mid(31, 28)));
     quint8 packed_cmd = Varicode::bitsToInt(Varicode::strToBits(bits.mid(59, 5)));
@@ -594,3 +599,56 @@ QStringList Varicode::unpackDirectedMessage(const QString &text){
 
     return unpacked;
 }
+
+QString Varicode::packDataMessage(const QString &text, int *n){
+    QString frame;
+
+    // [1][63],[5]
+    quint8 is_data = 1;
+    auto frameBits = (
+        Varicode::intToBits(is_data, 1)
+    );
+
+    int i = 0;
+    foreach(auto charBits, Varicode::huffEncode(text)){
+        if(frameBits.length() + charBits.length() < 63){
+            frameBits += charBits;
+            i++;
+            continue;
+        }
+        break;
+    }
+
+    int pad = 64 - frameBits.length();
+    if(pad){
+        frameBits += Varicode::intToBits(1, pad);
+    }
+
+    frame = Varicode::pack64bits(Varicode::bitsToInt(frameBits)) + Varicode::pack5bits(pad & 31);
+    *n = i;
+
+    return frame;
+}
+
+QString Varicode::unpackDataMessage(const QString &text){
+    QString unpacked;
+
+    if(text.length() < 13){
+        return unpacked;
+    }
+
+    auto bits = Varicode::intToBits(Varicode::unpack64bits(text.left(12)), 64);
+    quint8 pad = Varicode::unpack5bits(text.right(1));
+
+    quint8 is_data = (int)bits.at(0);
+    if(is_data != 1){
+        return unpacked;
+    }
+
+    // pop off the is_data bit
+    bits.removeAt(0);
+
+    unpacked = Varicode::huffDecode(bits, pad);
+
+    return unpacked;
+}

varicode.h

@@ -32,7 +32,7 @@ public:
 
     static QList<QVector<bool>> huffEncode(QString const& text);
     static QVector<bool> huffFlatten(QList<QVector<bool>> &list);
-    static QString huffDecode(QVector<bool> const& bitvec);
+    static QString huffDecode(QVector<bool> const& bitvec, int pad=0);
 
     static QVector<bool> bytesToBits(char * bitvec, int n);
     static QVector<bool> strToBits(QString const& bitvec);
@@ -63,6 +63,9 @@ public:
     static bool isCommandAllowed(const QString &cmd);
     static QString packDirectedMessage(QString const& text, QString const& callsign, int *n);
     static QStringList unpackDirectedMessage(QString const& text);
+
+    static QString packDataMessage(QString const& text, int *n);
+    static QString unpackDataMessage(QString const& text);
 };
 
 #endif // VARICODE_H