diff --git a/mainwindow.cpp b/mainwindow.cpp index 0ac0b75..dea0070 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -6096,10 +6096,10 @@ void MainWindow::on_extFreeTextMsgEdit_currentTextChanged (QString const& text) { QString x; QString::const_iterator i; - QSet validChars = Varicode::huffValidChars(); for(i = text.constBegin(); i != text.constEnd(); i++){ - auto ch = (*i).toUpper(); - if(validChars.contains(ch) || ch == '\n'){ + auto ch = (*i).toUpper().toLatin1(); + if(ch == 10 || (32 <= ch && ch <= 126)){ + // newline or printable 7-bit ascii x += ch; } } @@ -8770,7 +8770,7 @@ void MainWindow::updateFrameCountDisplay(QString text, int count){ auto words = text.split(" ", QString::SkipEmptyParts).length(); auto wpm = QString::number(words/(count/4.0), 'f', 1); - auto cpm = QString::number(text.length()/(count/4.0), 'f', 0); + auto cpm = QString::number(text.length()/(count/4.0), 'f', 1); wpm_label.setText(QString("%1wpm / %2cpm").arg(wpm).arg(cpm)); wpm_label.setVisible(true); } else { diff --git a/messagereplydialog.cpp b/messagereplydialog.cpp index ff88ffc..ee9288d 100644 --- a/messagereplydialog.cpp +++ b/messagereplydialog.cpp @@ -46,10 +46,10 @@ void MessageReplyDialog::on_textEdit_textChanged(){ QString x; QString::const_iterator i; - QSet validChars = Varicode::huffValidChars(); for(i = text.constBegin(); i != text.constEnd(); i++){ - auto ch = (*i).toUpper(); - if(validChars.contains(ch)){ + auto ch = (*i).toUpper().toLatin1(); + if(ch == 10 || (32 <= ch && ch <= 126)){ + // newline or printable 7-bit ascii x += ch; } } diff --git a/varicode.cpp b/varicode.cpp index 5930569..9b0d5b2 100644 --- a/varicode.cpp +++ b/varicode.cpp @@ -118,221 +118,50 @@ QRegularExpression compound_re("^\\s*[<]" + QMap hufftable = { // char code weight - // 3 bits - { " " , "000" }, // 1300 - { "E" , "001" }, // 1270.2 - - // 4 bits - { "T" , "1100" }, // 905.6 - { "A" , "1010" }, // 816.7 - { "O" , "0111" }, // 750.7 - { "I" , "0101" }, // 696.6 - { "N" , "0100" }, // 674.9 - - // 5 bits - { "S" , "11111" }, // 632.7 - { "H" , "11110" }, // 609.4 - { "R" , "11101" }, // 598.7 - { "D" , "10111" }, // 425.3 - { "L" , "10110" }, // 402.5 - - // 6 bits - { "C" , "111001" }, // 278.2 - { "U" , "111000" }, // 275.8 - { "M" , "110111" }, // 240.6 - { "W" , "110110" }, // 236.0 - { "F" , "110100" }, // 222.8 - { "G" , "100111" }, // 201.5 - { "Q" , "100110" }, // 200 - { "Y" , "011010" }, // 197.4 - { "P" , "011001" }, // 192.9 - { "B" , "011000" }, // 149.2 - - // 7 bits - { "\\" , "0110111" }, // 100 <- escape - { "." , "1000000" }, // 100 - { "0" , "1000001" }, // 100 - { "1" , "1000010" }, // 100 - { "2" , "1000011" }, // 100 - { "3" , "1000100" }, // 100 - { "4" , "1000101" }, // 100 - { "5" , "1000110" }, // 100 - { "6" , "1000111" }, // 100 - { "7" , "1001000" }, // 100 - { "8" , "1001001" }, // 100 - { "9" , "1001010" }, // 100 - { "?" , "1001011" }, // 100 - { "/" , "1101010" }, // 100 - { "V" , "0110110" }, // 97.8 - - // 8 bits - { "K" , "11010111" }, // 77.2 - - // 10 bits - { "J" , "1101011010" }, // 15.3 - { "X" , "1101011001" }, // 15.0 - - // 11 bits - { "Z" , "11010110110" }, // 7.4 - { ":" , "11010110000" }, // 5 - - // 12 bits - { "+" , "110101100011" }, // 5 - { "-" , "110101101110" }, // 5 - { "!" , "110101101111" }, // 5 - { "\x04" , "110101100010" }, // 1 <- eot - - /* - A-Z 0-9 Space \\ ? / : - + ! - special chars that are escaped will be added here too... - */ -}; - -/* -via https://www3.nd.edu/~busiforc/handouts/cryptography/Letter%20Frequencies.html#Most_common_trigrams_.28in_order.29 - -most common trigrams: -the = 12 bits -and = 13 bits -tha = 13 bits -ent = 11 bits -ing = 14 bits ** -ion = 12 bits -tio = 12 bits -for = 15 bits ** -nde = 12 bits -has = 14 bits -nce = 13 bits -edt = 12 bits -tis = 13 bits -oft = 14 bits -sth = 14 bits -men = 13 bits -her = 13 bits -hat = 13 bits -his = 14 bits -ere = 11 bits -ter = 12 bits -was = 15 bits -you = 16 bits ** -ith = 13 bits -ver = 15 bits -all = 14 bits -wit = 14 bits -thi = 13 bits - -most common quadgrams: -that = 17 bits -ther = 17 bits -with = 18 bits -tion = 16 bits -here = 16 bits -ould = 20 bits ** -ight = 19 bits -have = 19 bits -hich = 20 bits ** -whic = 21 bits ** -this = 18 bits -thin = 18 bits -they = 18 bits -atio = 16 bits -ever = 18 bits -from = 21 bits ** -ough = 21 bits ** -were = 17 bits -hing = 18 bits -ment = 17 bits - -potential contenders: -_DE_ = 14 bits -BTU = 16 bits -... = 21 bits -599 = 21 bits -FT8 = 17 bits -BAND = 19 bits -FT8CALL = 37 bits -DIPOLE = 27 bits -VERT = 19 bits -BEAM = 19 bits -*/ - - -/* -original: Space \\ ? / : - + ! -needed: ^,&@#$%'"()<>|*[]{}=;_~` -*/ -QMap huffescapes = { - - // 10 bits - { "\\ ", " DE " }, // 14 bits - 4 bit savings - { "\\E", "," }, - - // 11 bits - { "\\T", "&" }, - { "\\A", "@" }, - { "\\O", "#" }, - { "\\I", "$" }, - { "\\N", "%" }, - - // 12 bits - { "\\S", "\'" }, - { "\\H", "\"" }, - { "\\R", "(" }, - { "\\D", ")" }, - { "\\L", "|" }, - - // 13 bits - // trigram / quadgram efficiency - { "\\C", "YOU" }, // 16 bits - 3 bit savings - { "\\U", "THAT" }, // 17 bits - 4 bit savings - { "\\M", "THER" }, // 17 bits - 4 bit savings - { "\\W", "WITH" }, // 18 bits - 5 bit savings - { "\\F", "TION" }, // 16 bits - 3 bit savings - { "\\G", "HERE" }, // 16 bits - 3 bit savings - { "\\Q", "OULD" }, // 20 bits - 7 bit savings - { "\\Y", "IGHT" }, // 19 bits - 6 bit savings - { "\\P", "HAVE" }, // 19 bits - 6 bit savings - { "\\B", "HICH" }, // 20 bits - 7 bit savings - -#if 0 - // 14 bits - { "\\.", "" }, - { "\\0", "" }, -#endif - - { "\\1", "<" }, - { "\\2", ">" }, - { "\\3", "[" }, - { "\\4", "]" }, - { "\\5", "{" }, - { "\\6", "}" }, - { "\\7", "*" }, - { "\\8", "=" }, - { "\\9", ";" }, - { "\\?", "WHIC" }, // 21 bits - 7 bit savings - { "\\/", "THIS" }, // 18 bits - 4 bit savings - { "\\V", "FROM" }, // 21 bits - 7 bit savings - - // 15 bits - // quadgram efficiency - { "\\K" , "OUGH" }, // 21 bits - 6 bit savings - - // 17 bits -#if 0 - { "\\J" , "" }, - { "\\X" , "" }, -#endif - - // 18 bits - { "\\Z" , "^" }, - { "\\:" , "~" }, - - // 19 bits - { "\\+" , "`" }, - { "\\-" , "_" }, - - // special case :) - { "\\!" , "FT8CALL" }, // 37 bits - 18 bit savings + { " " , "01" }, // 1.0 + { "E" , "100" }, // 0.5 + { "T" , "1101" }, // 0.333333333333 + { "A" , "0011" }, // 0.25 + { "O" , "11111" }, // 0.2 + { "I" , "11100" }, // 0.166666666667 + { "N" , "10111" }, // 0.142857142857 + { "S" , "10100" }, // 0.125 + { "H" , "00011" }, // 0.111111111111 + { "R" , "00000" }, // 0.1 + { "D" , "111011" }, // 0.0909090909091 + { "L" , "110011" }, // 0.0833333333333 + { "C" , "110001" }, // 0.0769230769231 + { "U" , "101101" }, // 0.0714285714286 + { "M" , "101011" }, // 0.0666666666667 + { "W" , "001011" }, // 0.0625 + { "F" , "001001" }, // 0.0588235294118 + { "G" , "000101" }, // 0.0555555555556 + { "Y" , "000011" }, // 0.0526315789474 + { "P" , "1111011" }, // 0.05 + { "B" , "1111001" }, // 0.047619047619 + { "." , "1110100" }, // 0.0434782608696 + { "V" , "1100101" }, // 0.0416666666667 + { "K" , "1100100" }, // 0.04 + { "-" , "1100001" }, // 0.0384615384615 + { "+" , "1100000" }, // 0.037037037037 + { "?" , "1011001" }, // 0.0344827586207 + { "!" , "1011000" }, // 0.0333333333333 + {"\"" , "1010101" }, // 0.0322580645161 + { "X" , "1010100" }, // 0.03125 + { "0" , "0010101" }, // 0.030303030303 + { "J" , "0010100" }, // 0.0294117647059 + { "1" , "0010001" }, // 0.0285714285714 + { "Q" , "0010000" }, // 0.0277777777778 + { "2" , "0001001" }, // 0.027027027027 + { "Z" , "0001000" }, // 0.0263157894737 + { "3" , "0000101" }, // 0.025641025641 + { "5" , "0000100" }, // 0.025 + { "4" , "11110101" }, // 0.0243902439024 + { "9" , "11110100" }, // 0.0238095238095 + { "8" , "11110001" }, // 0.0232558139535 + { "6" , "11110000" }, // 0.0227272727273 + { "7" , "11101011" }, // 0.0222222222222 + { "/" , "11101010" }, // 0.0217391304348 }; QChar ESC = '\\'; // Escape char @@ -381,35 +210,6 @@ QMap dbm2mw = { {60 , 1000000}, // 1000W }; - -QMap initializeEscapes(QMap huff, QMap escapes){ - QMap newhuff(huff); - foreach(auto escapeString, escapes.keys()){ - auto ch = escapes[escapeString]; - auto encoded = Varicode::huffEncode(huff, escapeString); - QList> e; - foreach(auto pair, encoded){ - e.append(pair.second); - } - auto bits = Varicode::bitsListToBits(e); - newhuff[ch] = Varicode::bitsToStr(bits); - } - -#if PRINT_VARICODE_ALPHABET - auto keys = newhuff.keys(); - qSort(keys.begin(), keys.end(), [newhuff](QChar a, QChar b){ - return newhuff[a].length() < newhuff[b].length(); - }); - foreach(auto ch, keys){ - qDebug() << ch << newhuff[ch] << newhuff[ch].length(); - } -#endif - - return newhuff; -} - -QMap hufftableescaped = initializeEscapes(hufftable, huffescapes); - /* * UTILITIES */ @@ -467,10 +267,6 @@ QMap Varicode::defaultHuffTable(){ return hufftable; } -QMap Varicode::defaultHuffTableEscaped(){ - return hufftableescaped; -} - QString Varicode::cqString(int number){ if(!cqs.contains(number)){ return QString{}; @@ -630,39 +426,10 @@ QString Varicode::huffDecode(QMap const &huff, QVector c return text; } -QString Varicode::huffUnescape(QString const &input){ - QString text = input; - // unescape alternate alphabet - foreach(auto escaped, huffescapes.keys()){ - text = text.replace(escaped, huffescapes[escaped]); - } - return text; +QSet Varicode::huffValidChars(const QMap &huff){ + return QSet::fromList(huff.keys()); } -QString Varicode::huffEscape(QString const &input){ - QString text = input; - // escape alternate alphabet - foreach(auto unescaped, huffescapes.values()){ - text = text.replace(unescaped, huffescapes.key(unescaped)); - } - return text; -} - -QSet Varicode::huffValidChars(){ - return QSet::fromList(hufftableescaped.keys()); -} - -bool Varicode::huffShouldEscape(QString const &input){ - foreach(auto ch, huffescapes.values()){ - if(input.contains(ch)){ - return true; - } - } - - return false; -} - - // convert char* array of 0 bytes and 1 bytes to bool vector QVector Varicode::bytesToBits(char *bitvec, int n){ QVector bits; @@ -1343,7 +1110,7 @@ QString Varicode::packCompoundFrame(const QString &baseCallsign, const QString & QString frame; // needs to be a compound type... - if(type == FrameDataPadded || type == FrameDataUnpadded || type == FrameDirected){ + if(type == FrameDataCompressed || type == FrameDataUncompressed || type == FrameDirected){ return frame; } @@ -1390,7 +1157,7 @@ QStringList Varicode::unpackCompoundFrame(const QString &text, quint8 *pType, qu quint8 packed_flag = Varicode::bitsToInt(bits.mid(0, 3)); // needs to be a beacon type... - if(packed_flag == FrameDataPadded || packed_flag == FrameDataUnpadded || packed_flag == FrameDirected){ + if(packed_flag == FrameDataCompressed || packed_flag == FrameDataUncompressed || packed_flag == FrameDirected){ return unpacked; } @@ -1546,24 +1313,33 @@ QStringList Varicode::unpackDirectedMessage(const QString &text, quint8 *pType){ return unpacked; } -QString Varicode::packDataMessage(const QString &input, int *n){ +QString packHuffMessage(const QString &input, int *n){ static const int frameSize = 72; QString frame; -#if USE_HUFF_DATA_PACKING // [3][69] = 72 QVector frameDataBits; - QVector frameHeaderBits = Varicode::intToBits(FrameDataUnpadded, 3); + QVector frameHeaderBits = Varicode::intToBits(Varicode::FrameDataUncompressed, 3); int i = 0; - // we use the escaped table here, so they the escapes and the characters are packed together... - foreach(auto pair, Varicode::huffEncode(hufftableescaped, input)){ + // only pack huff messages that only contain valid chars + QString::const_iterator it; + QSet validChars = Varicode::huffValidChars(Varicode::defaultHuffTable()); + for(it = input.constBegin(); it != input.constEnd(); it++){ + auto ch = (*it).toUpper(); + if(!validChars.contains(ch)){ + return frame; + } + } + + // pack using the default huff table + foreach(auto pair, Varicode::huffEncode(Varicode::defaultHuffTable(), input)){ auto charN = pair.first; auto charBits = pair.second; - if(frameHeaderBits.length() + frameDataBits.length() + charBits.length() <= frameSize){ + if(frameHeaderBits.length() + frameDataBits.length() + charBits.length() < frameSize){ frameDataBits += charBits; i += charN; continue; @@ -1575,8 +1351,6 @@ QString Varicode::packDataMessage(const QString &input, int *n){ int pad = frameSize - frameHeaderBits.length() - frameDataBits.length(); if(pad){ - frameHeaderBits = Varicode::intToBits(FrameDataPadded, 3); - // the way we will pad is this... // set the bit after the frame to 0 and every bit after that a 1 // to unpad, seek from the end of the bits until you hit a zero... the rest is the actual frame. @@ -1585,17 +1359,27 @@ QString Varicode::packDataMessage(const QString &input, int *n){ } } + qDebug() << "Huff bits" << frameDataBits.length() << "chars" << i; + QVector allBits = frameHeaderBits + frameDataBits + framePadBits; quint64 value = Varicode::bitsToInt(allBits.constBegin(), 64); quint8 rem = (quint8)Varicode::bitsToInt(allBits.constBegin() + 64, 8); frame = Varicode::pack72bits(value, rem); - *n = i; -#else + if(n) *n = i; + + return frame; +} + +QString packCompressedMessage(const QString &input, int *n){ + static const int frameSize = 72; + + QString frame; + QVector frameBits; - frameBits.append(Varicode::intToBits(FrameDataPadded, 3)); + frameBits.append(Varicode::intToBits(Varicode::FrameDataCompressed, 3)); int i = 0; foreach(auto pair, JSC::compress(input)){ @@ -1611,6 +1395,8 @@ QString Varicode::packDataMessage(const QString &input, int *n){ break; } + qDebug() << "Compressed bits" << frameBits.length() - 3 << "chars" << i; + int pad = frameSize - frameBits.length(); if(pad){ // the way we will pad is this... @@ -1625,12 +1411,29 @@ QString Varicode::packDataMessage(const QString &input, int *n){ quint8 rem = (quint8)Varicode::bitsToInt(frameBits.constBegin() + 64, 8); frame = Varicode::pack72bits(value, rem); - *n = i; -#endif + if(n) *n = i; return frame; } +QString Varicode::packDataMessage(const QString &input, int *n){ + QString huffFrame; + int huffChars = 0; + huffFrame = packHuffMessage(input, &huffChars); + + QString compressedFrame; + int compressedChars = 0; + compressedFrame = packCompressedMessage(input, &compressedChars); + + if(huffChars > compressedChars){ + if(n) *n = huffChars; + return huffFrame; + } else { + if(n) *n = compressedChars; + return compressedFrame; + } +} + QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){ QString unpacked; @@ -1643,29 +1446,19 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){ auto bits = Varicode::intToBits(value, 64) + Varicode::intToBits(rem, 8); quint8 type = Varicode::bitsToInt(bits.mid(0, 3)); - if(type == FrameDataUnpadded){ - bits = bits.mid(3); - } else if(type == FrameDataPadded) { - int n = bits.lastIndexOf(0); - bits = bits.mid(3, n-3); - } else { - return unpacked; + + int n = bits.lastIndexOf(0); + bits = bits.mid(3, n-3); + + if(type == FrameDataUncompressed){ + // huff decode the bits (without escapes) + unpacked = Varicode::huffDecode(Varicode::defaultHuffTable(), bits); + if(pType) *pType = type; + } else if(type == FrameDataCompressed) { + unpacked = JSC::decompress(bits); + if(pType) *pType = type; } -#if USE_HUFF_DATA_PACKING - // huff decode the bits (without escapes) - unpacked = Varicode::huffDecode(hufftable, bits); - - // then... unescape special characters - unpacked = Varicode::huffUnescape(unpacked); - - if(pType) *pType = type; -#else - unpacked = JSC::decompress(bits); - - if(pType) *pType = type; -#endif - return unpacked; } @@ -1758,7 +1551,7 @@ QStringList Varicode::buildMessageFrames( bool dirToCompound = dirTo.contains("/"); int m = 0; - QString datFrame = Varicode::packDataMessage(line.left(24) + "\x04", &m); // 66 / 3 + 2 = 22 (maximum number of 3bit chars we could possibly stuff in here plus 2 for good measure :P) + QString datFrame = Varicode::packDataMessage(line, &m); // if this parses to a standard FT8 free text message // but it can be parsed as a directed message, then we diff --git a/varicode.h b/varicode.h index fcd9712..0a39e67 100644 --- a/varicode.h +++ b/varicode.h @@ -30,8 +30,8 @@ public: FrameCompoundDirected = 2, // [010] FrameDirected = 3, // [011] FrameReservedA = 4, // [100] <- Reserved for future use, likely an extension of one of these formats. - FrameDataUnpadded = 5, // [101] - FrameDataPadded = 6, // [110] + FrameDataUncompressed = 5, // [101] + FrameDataCompressed = 6, // [110] FrameReservedB = 7, // [111] <- Reserved for future use, likely binary data / other formats. }; @@ -44,8 +44,8 @@ public: "FrameCompoundDirected", "FrameDirected", "FrameReservedA", - "FrameDataUnpadded", - "FrameDataPadded", + "FrameDataUncompressed", + "FrameDataCompressed", "FrameReservedB" }; @@ -61,7 +61,6 @@ public: static QString lstrip(const QString& str); static QMap defaultHuffTable(); - static QMap defaultHuffTableEscaped(); static QString cqString(int number); static bool startsWithCQ(QString text); static QString formatSNR(int snr); @@ -78,11 +77,7 @@ public: static QList>> huffEncode(const QMap &huff, QString const& text); static QString huffDecode(const QMap &huff, QVector const& bitvec); - - static QString huffUnescape(QString const &input); - static QString huffEscape(QString const &input); - static QSet huffValidChars(); - static bool huffShouldEscape(QString const &input); + static QSet huffValidChars(const QMap &huff); static QVector bytesToBits(char * bitvec, int n); static QVector strToBits(QString const& bitvec);