Working through compression implementation for data frames

jsc.cpp

@@ -24,10 +24,10 @@
 #include <cmath>
 
 
-Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c){
+Codeword JSC::codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c){
     QList<Codeword> out;
 
-    quint32 v = ((index % s) << 1) + (int)separate;
+    quint32 v = ((index % s) << 1) + (quint32)separate;
     out.prepend(Varicode::intToBits(v, bytesize + 1));
 
     quint32 x = index / s;
@@ -45,78 +45,41 @@ Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, qui
     return word;
 }
 
-QPair<CompressionMap, CompressionList> JSC::loadCompressionTable(){
+QList<CodewordPair> JSC::compress(QString text){
-    CompressionMap out;
-    CompressionList words;
-
-    for(int i = 0; i < JSC::size; i++){
-        out[JSC::map[i]] = i;
-    }
-
-    words.reserve(JSC::size);
-    for(int i = 0; i < JSC::size; i++){
-        words.append(JSC::list[i]);
-    }
-
-    return {out, words};
-}
-
-QPair<CompressionMap, CompressionList> JSC::loadCompressionTable(QTextStream &stream){
-    CompressionMap out;
-    CompressionList words;
-
-    int index = 0;
-
-    while(!stream.atEnd()){
-        // assume that this is in sorted order, that each word is already upper case.
-        auto word = stream.readLine().trimmed();
-        if(out.contains(word)){
-            continue;
-        }
-        out[word] = index;
-        words.append(word);
-        index++;
-    }
-
-#if 1
-    qStableSort(words.begin(), words.end(), [out](QString const &left, QString const &right){
-        return out[left] < out[right];
-    });
-    qStableSort(words.begin(), words.end(), [](QString const &left, QString const &right){
-        if(left.length() == right.length()){
-            return left < right;
-        }
-        return left.length() < right.length();
-    });
-#endif
-
-    return {out, words};
-}
-
-QList<CodewordPair> JSC::compress(CompressionTable table, QString text){
     QList<CodewordPair> out;
 
     const quint32 b = 4;
     const quint32 s = 7;
     const quint32 c = pow(2, 4) - s;
 
-    auto map = table.first;
-    auto list = table.second;
-
     foreach(QString w, text.split(" ", QString::SkipEmptyParts)){
-        if(map.contains(w)){
+        bool ok = false;
-            auto index = map[w];
+        auto index = lookup(w, &ok);
-            out.append({ codeword(index, true, b, s, c), w.length() });
+        if(ok){
+            // cool, we found the word...
+            out.append({ codeword(index, true, b, s, c), (quint32)w.length() + 1 /* for the space that follows */ });
         } else {
-            // prefix match?
+            // hmm. no dice. let's go for a prefix match
             while(!w.isEmpty()){
                 bool hasPrefix = false;
+                auto d = w.toLatin1().data();
+                for(quint32 i = 0; i < JSC::size; i++){
+
+                    // TODO: we could probably precompute these sizes
+                    quint32 len = strlen(JSC::list[i]);
+
+                    if(strncmp(d, JSC::list[i], len) == 0){
+                        w = QString(w.mid(len));
+
+                        auto word = JSC::list[i];
+                        auto index = lookup(word, &ok);
+                        if(ok){
+                            bool isLast = w.isEmpty();
+                            out.append({ codeword(index, isLast, b, s, c), len + (isLast ? 1 : 0) /* for the space that follows */});
+                            hasPrefix = true;
+                            break;
+                        }
 
-                foreach(QString word, list){
-                    if(w.startsWith(word)){
-                        w = QString(w.mid(word.length()));
-                        out.append({ codeword(map[word], w.isEmpty(), b, s, c), word.length()});
-                        hasPrefix = true;
                         break;
                     }
                 }
@@ -132,7 +95,7 @@ QList<CodewordPair> JSC::compress(CompressionTable table, QString text){
     return out;
 }
 
-QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
+QString JSC::decompress(Codeword const& bitvec){
     const quint32 b = 4;
     const quint32 s = 7;
     const quint32 c = pow(2, b) - s;
@@ -177,7 +140,7 @@ QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
 
         j = j*s + bytes[start + k] + base[k];
 
-        out.append(table.first.key(j));
+        out.append(QString(JSC::map[j])); // table.first.key(j));
         if(separators.first() == start + k){
             out.append(" ");
             separators.removeFirst();
@@ -188,3 +151,19 @@ QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
 
     return out.join("");
 }
+
+quint32 JSC::lookup(QString w, bool * ok){
+    return lookup(w.toLatin1().data(), ok);
+}
+
+quint32 JSC::lookup(char const* b, bool *ok){
+    for(quint32 i = 0; i < JSC::size; i++){
+        if(strcmp(b, JSC::map[i]) == 0){
+            if(ok) *ok = true;
+            return i;
+        }
+    }
+
+    if(ok) *ok = false;
+    return 0;
+}

jsc.h

@@ -12,21 +12,25 @@
 #include <QPair>
 #include <QVector>
 
-typedef QMap<QString, int> CompressionMap;             // Map(Word, I) where I is the word index sorted by frequency
+typedef QPair<QVector<bool>, quint32> CodewordPair;        // Tuple(Codeword, N) where N = number of characters
-typedef QList<QString> CompressionList;                // List(Word) where list is sorted
+typedef QVector<bool> Codeword;                        // Codeword bit vector
-typedef QPair<CompressionMap, CompressionList> CompressionTable; // Tuple(Map, List)
+
-typedef QPair<QVector<bool>, int> CodewordPair;        // Tuple(Codeword, N) where N = number of characters
-typedef QVector<bool> Codeword;
 
 class JSC
 {
 public:
+#if 0
     static CompressionTable loadCompressionTable();
     static CompressionTable loadCompressionTable(QTextStream &stream);
-    static QList<CodewordPair> compress(CompressionTable table, QString text);
+#endif
-    static QString decompress(CompressionTable table, Codeword const& bits);
+    static Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c);
+    static QList<CodewordPair> compress(QString text);
+    static QString decompress(Codeword const& bits);
 
-    static const int size = 233638;
+    static quint32 lookup(QString w, bool *ok);
+    static quint32 lookup(char const* b, bool *ok);
+
+    static const quint32 size = 233638;
     static char const* map[];
     static char const* list[];
 };

mainwindow.cpp

@@ -1450,18 +1450,6 @@ void MainWindow::not_GA_warning_message ()
 }
 
 void MainWindow::initializeDummyData(){
-
-    auto table = JSC::loadCompressionTable();
-
-    Codeword bits;
-    auto compressed = JSC::compress(table, "E A T EAT TEA ATE EATTET");
-    foreach(auto pair, compressed){
-        qDebug() << "compressed" << Varicode::bitsToStr(pair.first);
-        bits.append(pair.first);
-    }
-
-    qDebug() << "decomressed" << JSC::decompress(table, bits);
-
     if(!QApplication::applicationName().contains("dummy")){
         return;
     }
@@ -2773,6 +2761,11 @@ void MainWindow::createStatusBar()                           //createStatusBar
   progressBar.setMinimumSize (QSize {100, 18});
   progressBar.setFormat ("%v/%m");
 
+  statusBar()->addPermanentWidget(&wpm_label);
+  wpm_label.setMinimumSize (QSize {90, 18});
+  wpm_label.setFrameStyle (QFrame::Panel | QFrame::Sunken);
+  wpm_label.setAlignment(Qt::AlignHCenter);
+
   statusBar ()->addPermanentWidget (&watchdog_label);
   update_watchdog_label ();
 }
@@ -6134,15 +6127,6 @@ void MainWindow::on_extFreeTextMsgEdit_currentTextChanged (QString const& text)
 
       ui->extFreeTextMsgEdit->setTextCursor(c);
     }
-
-    int count = countFT8MessageFrames(x);
-    if(count > 0){
-        ui->startTxButton->setText(QString("Send (%1)").arg(count));
-        ui->startTxButton->setEnabled(true);
-    } else {
-        ui->startTxButton->setText("Send");
-        ui->startTxButton->setEnabled(false);
-    }
 }
 
 int MainWindow::currentFreqOffset(){
@@ -9009,6 +8993,25 @@ void MainWindow::updateButtonDisplay(){
         int count = m_txFrameCount;
         int sent = count - m_txFrameQueue.count();
         ui->startTxButton->setText(m_tune ? "Tuning" : QString("Sending (%1/%2)").arg(sent).arg(count));
+    } else {
+
+        // TODO: only if text changed
+
+        auto text = ui->extFreeTextMsgEdit->toPlainText();
+        int count = countFT8MessageFrames(text);
+        if(count > 0){
+            ui->startTxButton->setText(QString("Send (%1)").arg(count));
+            ui->startTxButton->setEnabled(true);
+
+            auto words = text.split(" ", QString::SkipEmptyParts).length();
+            auto wpm = QString::number(words/(count/4.0), 'g', 2);
+            wpm_label.setText(QString("%1 wpm").arg(wpm));
+        } else {
+            ui->startTxButton->setText("Send");
+            ui->startTxButton->setEnabled(false);
+            wpm_label.clear();
+        }
+
     }
 }
 

mainwindow.h

@@ -620,6 +620,7 @@ private:
   QLabel band_hopping_label;
   QProgressBar progressBar;
   QLabel watchdog_label;
+  QLabel wpm_label;
 
   QFuture<void> m_wav_future;
   QFutureWatcher<void> m_wav_future_watcher;
@@ -934,6 +935,7 @@ private:
   void write_transmit_entry (QString const& file_name);
 };
 
+
 extern int killbyname(const char* progName);
 extern void getDev(int* numDevices,char hostAPI_DeviceName[][50],
                    int minChan[], int maxChan[],

varicode.cpp

@@ -26,6 +26,7 @@
 #include "crc.h"
 
 #include "varicode.h"
+#include "jsc.h"
 
 const int nalphabet = 41;
 QString alphabet = {"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ+-./?"}; // alphabet to encode _into_ for FT8 freetext transmission
@@ -1527,7 +1528,42 @@ QString Varicode::packDataMessage(const QString &input, int *n){
     static const int frameSize = 72;
 
     QString frame;
+    QVector<bool> frameBits;
 
+    frameBits.append(Varicode::intToBits(FrameDataPadded, 3));
+
+    int i = 0;
+    foreach(auto pair, JSC::compress(input)){
+        auto bits = pair.first;
+        auto chars = pair.second;
+
+        if(frameBits.length() + bits.length() < frameSize){
+            frameBits.append(bits);
+            i += chars;
+            continue;
+        }
+
+        break;
+    }
+
+    int pad = frameSize - frameBits.length();
+    if(pad){
+        // the way we will pad is this...
+        // set the bit after the frame to 0 and every bit after that a 1
+        // to unpad, seek from the end of the bits until you hit a zero... the rest is the actual frame.
+        for(int i = 0; i < pad; i++){
+            frameBits.append(i == 0 ? (bool)0 : (bool)1);
+        }
+    }
+
+    quint64 value = Varicode::bitsToInt(frameBits.constBegin(), 64);
+    quint8 rem = (quint8)Varicode::bitsToInt(frameBits.constBegin() + 64, 8);
+    frame = Varicode::pack72bits(value, rem);
+
+    *n = i;
+
+
+#if 0
     // [3][69] = 72
     QVector<bool> frameDataBits;
 
@@ -1568,6 +1604,7 @@ QString Varicode::packDataMessage(const QString &input, int *n){
     frame = Varicode::pack72bits(value, rem);
 
     *n = i;
+#endif
 
     return frame;
 }
@@ -1583,6 +1620,7 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){
     quint64 value = Varicode::unpack72bits(text, &rem);
     auto bits = Varicode::intToBits(value, 64) + Varicode::intToBits(rem, 8);
 
+#if 0
     quint8 type = Varicode::bitsToInt(bits.mid(0, 3));
     if(type == FrameDataUnpadded){
         bits = bits.mid(3);
@@ -1600,6 +1638,7 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){
     unpacked = Varicode::huffUnescape(unpacked);
 
     if(pType) *pType = type;
+#endif
 
     return unpacked;
 }