From de66664635e958f82df6baa0c09c8200855c4ebb Mon Sep 17 00:00:00 2001 From: Jordan Sherer Date: Thu, 26 Jul 2018 12:47:03 -0400 Subject: [PATCH] Added extended alphabet for special characters --- mainwindow.cpp | 9 +- varicode.cpp | 253 ++++++++++++++++++++++++++++++++++++------------- varicode.h | 12 ++- 3 files changed, 201 insertions(+), 73 deletions(-) diff --git a/mainwindow.cpp b/mainwindow.cpp index f5a19b9..46ec7ce 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -164,7 +164,7 @@ namespace { Radio::Frequency constexpr default_frequency {14074000}; QRegExp message_alphabet {"[- A-Za-z0-9+./?:!^]*"}; - QRegExp message_input_alphabet {"[- A-Za-z0-9+./?\\n:!^@&|$%]*"}; // @&|$% are used for commands but are never transmitted + QRegExp message_input_alphabet {"[- A-Za-z0-9+./?\\n:!^,&@#$%*()<>'\"|=]*"}; // grid exact match excluding RR73 QRegularExpression grid_regexp {"\\A(?![Rr]{2}73)[A-Ra-r]{2}[0-9]{2}([A-Xa-x]{2}){0,1}\\z"}; @@ -5637,7 +5637,9 @@ QPair MainWindow::buildFT8MessageFrames(QString const& QString dirFrame = Varicode::packDirectedMessage(line, basecall, &dirCmd, &n); int m = 0; - QString datFrame = Varicode::packDataMessage(line.left(21) + "\x04", &m); // 63 / 3 = 21 (maximum number of 3bit chars we could possibly stuff in here) + // packDataMessage can output a new line (huff escaping special characters) + QString datLineOut; + QString datFrame = Varicode::packDataMessage(line.left(21) + "\x04", &datLineOut, &m); // 63 / 3 = 21 (maximum number of 3bit chars we could possibly stuff in here) // if this parses to a standard FT8 free text message // but it can be parsed as a directed message, then we @@ -5649,6 +5651,9 @@ QPair MainWindow::buildFT8MessageFrames(QString const& } else if ((isFree || hasDirected) && m > 0) { useDat = true; frame = datFrame; + if(!datLineOut.isEmpty()){ + line = datLineOut; + } } else { useStd = true; frame = stdFrame; diff --git a/varicode.cpp b/varicode.cpp index 21315fb..018d0e0 100644 --- a/varicode.cpp +++ b/varicode.cpp @@ -74,59 +74,110 @@ QRegularExpression directed_re("^" "(?\\s?[-+]?(?:3[01]|[0-2]?[0-9]))?" ); -QMap huff = { +QMap hufftable = { // char code weight - {' ' , "000" }, // 1300 - {'E' , "001" }, // 1270.2 - {'T' , "1100" }, // 905.6 - {'A' , "1010" }, // 816.7 - {'O' , "0111" }, // 750.7 - {'I' , "0101" }, // 696.6 - {'N' , "0100" }, // 674.9 - {'S' , "11111" }, // 632.7 - {'H' , "11110" }, // 609.4 - {'R' , "11101" }, // 598.7 - {'D' , "10111" }, // 425.3 - {'L' , "10110" }, // 402.5 - {'C' , "111001" }, // 278.2 - {'U' , "111000" }, // 275.8 - {'M' , "110111" }, // 240.6 - {'W' , "110110" }, // 236.0 - {'F' , "110100" }, // 222.8 - {'G' , "100111" }, // 201.5 - {'Q' , "100110" }, // 200 - {'Y' , "011010" }, // 197.4 - {'P' , "011001" }, // 192.9 - {'B' , "011000" }, // 149.2 - {'!' , "0110111" }, // 100 - {'.' , "1000000" }, // 100 - {'0' , "1000001" }, // 100 - {'1' , "1000010" }, // 100 - {'2' , "1000011" }, // 100 - {'3' , "1000100" }, // 100 - {'4' , "1000101" }, // 100 - {'5' , "1000110" }, // 100 - {'6' , "1000111" }, // 100 - {'7' , "1001000" }, // 100 - {'8' , "1001001" }, // 100 - {'9' , "1001010" }, // 100 - {'?' , "1001011" }, // 100 - {'^' , "1101010" }, // 100 <- shift - {'V' , "0110110" }, // 97.8 - {'K' , "11010111" }, // 77.2 - {'J' , "1101011010" }, // 15.3 - {'X' , "1101011001" }, // 15.0 - {'Z' , "11010110110" }, // 7.4 - {':' , "11010110000" }, // 5 - {'+' , "110101100011" }, // 5 - {'-' , "110101101110" }, // 5 - {'/' , "110101101111" }, // 5 - {'\x04' , "110101100010" }, // 1 <- eot + { ' ' , "000" }, // 1300 + { 'E' , "001" }, // 1270.2 + { 'T' , "1100" }, // 905.6 + { 'A' , "1010" }, // 816.7 + { 'O' , "0111" }, // 750.7 + { 'I' , "0101" }, // 696.6 + { 'N' , "0100" }, // 674.9 + { 'S' , "11111" }, // 632.7 + { 'H' , "11110" }, // 609.4 + { 'R' , "11101" }, // 598.7 + { 'D' , "10111" }, // 425.3 + { 'L' , "10110" }, // 402.5 + { 'C' , "111001" }, // 278.2 + { 'U' , "111000" }, // 275.8 + { 'M' , "110111" }, // 240.6 + { 'W' , "110110" }, // 236.0 + { 'F' , "110100" }, // 222.8 + { 'G' , "100111" }, // 201.5 + { 'Q' , "100110" }, // 200 + { 'Y' , "011010" }, // 197.4 + { 'P' , "011001" }, // 192.9 + { 'B' , "011000" }, // 149.2 + { '\\' , "0110111" }, // 100 <- escape + { '.' , "1000000" }, // 100 + { '0' , "1000001" }, // 100 + { '1' , "1000010" }, // 100 + { '2' , "1000011" }, // 100 + { '3' , "1000100" }, // 100 + { '4' , "1000101" }, // 100 + { '5' , "1000110" }, // 100 + { '6' , "1000111" }, // 100 + { '7' , "1001000" }, // 100 + { '8' , "1001001" }, // 100 + { '9' , "1001010" }, // 100 + { '?' , "1001011" }, // 100 + { '/' , "1101010" }, // 100 + { 'V' , "0110110" }, // 97.8 + { 'K' , "11010111" }, // 77.2 + { 'J' , "1101011010" }, // 15.3 + { 'X' , "1101011001" }, // 15.0 + { 'Z' , "11010110110" }, // 7.4 + { ':' , "11010110000" }, // 5 + { '+' , "110101100011" }, // 5 + { '-' , "110101101110" }, // 5 + { '!' , "110101101111" }, // 5 + { '\x04' , "110101100010" }, // 1 <- eot - // A-Z 0-9 Space . ! ? ^ : + - / + /* + A-Z 0-9 Space . ! ? : + - / \\ + special chars that are escaped will be added here too... + */ }; -QChar huffeot = '\x04'; +/* +original: space + - / ? . ! : \\ +needed: ^,&@#$%*()<>'"|={}[];_~` +*/ +QMap huffescapes = { + { "\\ ", '^' }, + { "\\E", ',' }, + { "\\T", '&' }, + { "\\A", '@' }, + { "\\O", '#' }, + { "\\I", '$' }, + { "\\N", '%' }, + { "\\S", '\'' }, + { "\\H", '\"' }, + { "\\R", '(' }, + { "\\D", ')' }, + { "\\L", '<' }, + { "\\C", '>' }, + { "\\U", '|' }, + { "\\M", '*' }, + { "\\W", '[' }, + { "\\F", ']' }, + { "\\G", '{' }, + { "\\Q", '}' }, + { "\\Y", '=' }, + { "\\P", ';' }, + { "\\B", '_' }, + { "\\.", '~' }, + { "\\0", '`' }, + +#if 0 + // reserved <= 14 bits + { "\\1", '' }, + { "\\2", '' }, + { "\\3", '' }, + { "\\4", '' }, + { "\\5", '' }, + { "\\6", '' }, + { "\\7", '' }, + { "\\8", '' }, + { "\\9", '' }, + { "\\?", '' }, + { "\\/", '' }, + { "\\V", '' }, +#endif +}; + +QChar ESC = '\\'; // Escape char +QChar EOT = '\x04'; // EOT char quint32 nbasecall = 37 * 36 * 10 * 27 * 27 * 27; @@ -158,6 +209,35 @@ QMap dbm2mw = { {60 , 1000000}, // 1000W }; + +QMap initializeEscapes(QMap huff, QMap escapes){ + QMap newhuff(huff); + foreach(auto escapeString, escapes.keys()){ + auto ch = escapes[escapeString]; + auto encoded = Varicode::huffEncode(huff, escapeString); + auto bits = Varicode::bitsListToBits(encoded); + newhuff[ch] = Varicode::bitsToStr(bits); + } + +#if PRINT_VARICODE_ALPHABET + auto keys = newhuff.keys(); + qSort(keys.begin(), keys.end(), [newhuff](QChar a, QChar b){ + return newhuff[a].length() < newhuff[b].length(); + }); + foreach(auto ch, keys){ + qDebug() << ch << newhuff[ch] << newhuff[ch].length(); + } +#endif + + return newhuff; +} + +QMap hufftableescaped = initializeEscapes(hufftable, huffescapes); + +/* + * UTILITIES + */ + int mwattsToDbm(int mwatts){ int dbm = 0; auto values = dbm2mw.values(); @@ -182,6 +262,10 @@ int dbmTomwatts(int dbm){ return iter.value(); } +/* + * VARICODE + */ + QString Varicode::formatSNR(int snr){ if(snr < -60 || snr > 60){ return QString(); @@ -272,7 +356,7 @@ QStringList Varicode::parseGrids(const QString &input){ return grids; } -QList> Varicode::huffEncode(QString const& text){ +QList> Varicode::huffEncode(QMap const &huff, QString const& text){ QList> out; foreach(auto ch, text){ @@ -285,16 +369,8 @@ QList> Varicode::huffEncode(QString const& text){ return out; } -QVector Varicode::huffFlatten(QList> &list){ - QVector out; - foreach(auto vec, list){ - out += vec; - } - return out; -} - -QString Varicode::huffDecode(QVector const& bitvec, int pad){ - QString out; +QString Varicode::huffDecode(QMap const &huff, QVector const& bitvec, int pad){ + QString text; QString bits = bitsToStr(bitvec).mid(0, bitvec.length()-pad); @@ -303,12 +379,12 @@ QString Varicode::huffDecode(QVector const& bitvec, int pad){ bool found = false; foreach(auto key, huff.keys()){ if(bits.startsWith(huff[key])){ - if(key == huffeot){ - out.append(" "); + if(key == EOT){ + text.append(" "); found = false; break; } - out.append(key); + text.append(key); bits = bits.mid(huff[key].length()); found = true; } @@ -318,9 +394,38 @@ QString Varicode::huffDecode(QVector const& bitvec, int pad){ } } - return out; + return text; } +QString Varicode::huffUnescape(QString const &input){ + QString text = input; + // unescape alternate alphabet + foreach(auto escaped, huffescapes.keys()){ + text = text.replace(escaped, huffescapes[escaped]); + } + return text; +} + +QString Varicode::huffEscape(QString const &input){ + QString text = input; + // escape alternate alphabet + foreach(auto unescaped, huffescapes.values()){ + text = text.replace(unescaped, huffescapes.key(unescaped)); + } + return text; +} + +bool Varicode::huffShouldEscape(QString const &input){ + foreach(auto ch, huffescapes.values()){ + if(input.contains(ch)){ + return true; + } + } + + return false; +} + + // convert char* array of 0 bytes and 1 bytes to bool vector QVector Varicode::bytesToBits(char *bitvec, int n){ QVector bits; @@ -382,6 +487,14 @@ quint64 Varicode::bitsToInt(QVector::ConstIterator start, int n){ return v; } +QVector Varicode::bitsListToBits(QList> &list){ + QVector out; + foreach(auto vec, list){ + out += vec; + } + return out; +} + quint8 Varicode::unpack5bits(QString const& value){ return alphabet.indexOf(value.at(0)); } @@ -882,7 +995,7 @@ QStringList Varicode::unpackDirectedMessage(const QString &text){ return unpacked; } -QString Varicode::packDataMessage(const QString &text, int *n){ +QString Varicode::packDataMessage(const QString &input, QString * out, int *n){ QString frame; // [1][63],[5] = 69 @@ -892,7 +1005,9 @@ QString Varicode::packDataMessage(const QString &text, int *n){ ); int i = 0; - foreach(auto charBits, Varicode::huffEncode(text)){ + + // we use the escaped table here, so they the escapes and the characters are packed together... + foreach(auto charBits, Varicode::huffEncode(hufftableescaped, input)){ if(frameBits.length() + charBits.length() < 63){ frameBits += charBits; i++; @@ -930,7 +1045,11 @@ QString Varicode::unpackDataMessage(const QString &text){ // pop off the is_data bit bits.removeAt(0); - unpacked = Varicode::huffDecode(bits, pad); + // huff decode the bits (without escapes) + unpacked = Varicode::huffDecode(hufftable, bits, pad); + + // then... unescape special characters + unpacked = Varicode::huffUnescape(unpacked); return unpacked; } diff --git a/varicode.h b/varicode.h index 75b4925..8b07eab 100644 --- a/varicode.h +++ b/varicode.h @@ -39,9 +39,12 @@ public: static QStringList parseCallsigns(QString const &input); static QStringList parseGrids(QString const &input); - static QList> huffEncode(QString const& text); - static QVector huffFlatten(QList> &list); - static QString huffDecode(QVector const& bitvec, int pad=0); + static QList> huffEncode(const QMap &huff, QString const& text); + static QString huffDecode(const QMap &huff, QVector const& bitvec, int pad=0); + + static QString huffUnescape(QString const &input); + static QString huffEscape(QString const &input); + static bool huffShouldEscape(QString const &input); static QVector bytesToBits(char * bitvec, int n); static QVector strToBits(QString const& bitvec); @@ -50,6 +53,7 @@ public: static QVector intToBits(quint64 value, int expected=0); static quint64 bitsToInt(QVector const value); static quint64 bitsToInt(QVector::ConstIterator start, int n); + static QVector bitsListToBits(QList> &list); static quint8 unpack5bits(QString const& value); static QString pack5bits(quint8 packed); @@ -84,7 +88,7 @@ public: static QString packDirectedMessage(QString const& text, QString const& callsign, QString * pCmd, int *n); static QStringList unpackDirectedMessage(QString const& text); - static QString packDataMessage(QString const& text, int *n); + static QString packDataMessage(QString const& text, QString *out, int *n); static QString unpackDataMessage(QString const& text); };