Restructured data frame packing so we can send more over the wire in fewer frames

This commit is contained in:
Jordan Sherer 2018-10-29 03:26:10 -04:00
parent 38fc98702b
commit 440311a75b
6 changed files with 102 additions and 74 deletions

View File

@ -24,6 +24,7 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString
, frameType_(Varicode::FrameUnknown)
, isHeartbeat_(false)
, isAlt_(false)
, bits_{0}
{
if(message_.length() >= 1) {
message_ = message_.left (21).remove (QRegularExpression {"[<>]"});
@ -58,14 +59,17 @@ DecodedText::DecodedText (QString const& the_string, bool contest_mode, QString
}
}
bits_ = bits();
tryUnpack();
}
DecodedText::DecodedText (QString const& js8callmessage):
DecodedText::DecodedText (QString const& js8callmessage, int bits):
frameType_(Varicode::FrameUnknown),
message_(js8callmessage),
isHeartbeat_(false),
isAlt_(false)
isAlt_(false),
bits_(bits)
{
is_standard_ = QRegularExpression("^(CQ|DE|QRZ)\\s").match(message_).hasMatch();
@ -79,6 +83,10 @@ bool DecodedText::tryUnpack(){
}
bool unpacked = false;
if(!unpacked){
unpacked = tryUnpackData();
}
if(!unpacked){
unpacked = tryUnpackHeartbeat();
}
@ -91,10 +99,6 @@ bool DecodedText::tryUnpack(){
unpacked = tryUnpackDirected();
}
if(!unpacked){
unpacked = tryUnpackData();
}
return unpacked;
}
@ -212,6 +216,10 @@ bool DecodedText::tryUnpackData(){
return false;
}
if((bits_ & Varicode::JS8CallData) != Varicode::JS8CallData){
return false;
}
quint8 type = Varicode::FrameUnknown;
QString data = Varicode::unpackDataMessage(m, &type);

View File

@ -31,7 +31,7 @@ class DecodedText
{
public:
explicit DecodedText (QString const& message, bool, QString const& my_grid);
explicit DecodedText (QString const& js8callmessage);
explicit DecodedText (QString const& js8callmessage, int bits);
bool tryUnpack();
bool tryUnpackHeartbeat();
@ -109,6 +109,7 @@ private:
bool contest_mode_;
QString message_;
bool is_standard_;
int bits_;
};
#endif // DECODEDTEXT_H

View File

@ -2204,7 +2204,7 @@ void MainWindow::fastSink(qint64 frames)
m_config.color_NewCall(),m_config.ppfx());
m_bDecoded=true;
if (m_mode != "ISCAT") postDecode (true, decodedtext.string ());
writeAllTxt(message);
writeAllTxt(message, decodedtext.bits());
bool stdMsg = decodedtext.report(m_baseCall,
Radio::base_callsign(ui->dxCallEntry->text()),m_rptRcvd);
//if (stdMsg) pskPost (decodedtext);
@ -3445,7 +3445,7 @@ void::MainWindow::fast_decode_done()
m_bDecoded=true;
}
postDecode (true, decodedtext.string ());
writeAllTxt(message);
writeAllTxt(message, decodedtext.bits());
if(m_mode=="JT9" or m_mode=="MSK144") {
// find and extract any report for myCall
@ -3462,7 +3462,7 @@ void::MainWindow::fast_decode_done()
m_bFastDone=false;
}
void MainWindow::writeAllTxt(QString message)
void MainWindow::writeAllTxt(QString message, int bits)
{
// Write decoded text to file "ALL.TXT".
QFile f {m_config.writeable_data_dir ().absoluteFilePath ("ALL.TXT")};
@ -3474,7 +3474,7 @@ void MainWindow::writeAllTxt(QString message)
<< m_mode << endl;
m_RxLog=0;
}
auto dt = DecodedText(message);
auto dt = DecodedText(message, bits);
out << dt.message() << endl;
f.close();
} else {
@ -3569,7 +3569,7 @@ void MainWindow::readFromStdout() //readFromStdout
(bits == Varicode::JS8Call ||
((bits & Varicode::JS8CallFirst) == Varicode::JS8CallFirst) ||
((bits & Varicode::JS8CallLast) == Varicode::JS8CallLast) ||
((bits & Varicode::JS8CallReserved) == 0 /*Varicode::JS8CallReserved*/)) // This is unused...so is invalid at this time...
((bits & Varicode::JS8CallData) == Varicode::JS8CallData)) // This is unused...so is invalid at this time...
);
qDebug() << "valid" << bValidFrame << "decoded text" << decodedtext.message();
@ -4023,6 +4023,7 @@ void MainWindow::guiUpdate()
static quint64 lastLoop;
static char message[29];
static char msgsent[29];
static int msgibits;
double txDuration;
QString rt;
@ -4170,11 +4171,12 @@ void MainWindow::guiUpdate()
char ft8msgbits[75 + 12]; //packed 75 bit ft8 message plus 12-bit CRC
genft8_(message, MyGrid, &bcontest, &m_i3bit, msgsent, const_cast<char *> (ft8msgbits),
const_cast<int *> (itone), 22, 6, 22);
msgibits = m_i3bit;
msgsent[22]=0;
}
m_currentMessage = QString::fromLatin1(msgsent);
m_currentMessageBits = msgibits;
m_bCallingCQ = CALLING == m_QSOProgress
|| m_currentMessage.contains (QRegularExpression {"^(CQ|QRZ) "});
if(m_mode=="FT8") {
@ -4322,7 +4324,7 @@ void MainWindow::guiUpdate()
if(m_transmitting) {
char s[41];
auto dt = DecodedText(msgsent);
auto dt = DecodedText(msgsent, msgibits);
sprintf(s,"Tx: %s", dt.message().toLocal8Bit().mid(0, 41).data());
m_nsendingsh=0;
if(s[4]==64) m_nsendingsh=1;
@ -4486,7 +4488,7 @@ void MainWindow::startTx2()
void MainWindow::stopTx()
{
Q_EMIT endTransmitMessage ();
auto dt = DecodedText(m_currentMessage.trimmed());
auto dt = DecodedText(m_currentMessage.trimmed(), m_currentMessageBits);
last_tx_label.setText("Last Tx: " + dt.message()); //m_currentMessage.trimmed());
m_btxok = false;
@ -5301,7 +5303,7 @@ void MainWindow::createMessageTransmitQueue(QString const& text){
QStringList lines;
foreach(auto frame, frames){
auto dt = DecodedText(frame);
auto dt = DecodedText(frame.first, frame.second);
lines.append(dt.message());
}
@ -5324,9 +5326,9 @@ void MainWindow::resetMessageTransmitQueue(){
m_txMessageQueue.clear();
}
QString MainWindow::popMessageFrame(){
QPair<QString, int> MainWindow::popMessageFrame(){
if(m_txFrameQueue.isEmpty()){
return QString();
return QPair<QString, int>{};
}
return m_txFrameQueue.dequeue();
}
@ -5371,7 +5373,7 @@ int MainWindow::currentFreqOffset(){
return ui->RxFreqSpinBox->value();
}
QStringList MainWindow::buildMessageFrames(const QString &text){
QList<QPair<QString, int>> MainWindow::buildMessageFrames(const QString &text){
// prepare selected callsign for directed message
QString selectedCall = callsignSelected();
@ -5395,7 +5397,7 @@ QStringList MainWindow::buildMessageFrames(const QString &text){
#if 0
qDebug() << "frames:";
foreach(auto frame, frames){
auto dt = DecodedText(frame);
auto dt = DecodedText(frame.frame, frame.bits);
qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType());
}
#endif
@ -5407,7 +5409,10 @@ bool MainWindow::prepareNextMessageFrame()
{
m_i3bit = Varicode::JS8Call;
QString frame = popMessageFrame();
QPair<QString, int> f = popMessageFrame();
auto frame = f.first;
auto bits = f.second;
if(frame.isEmpty()){
ui->nextFreeTextMsg->clear();
updateTxButtonDisplay();
@ -5416,6 +5421,7 @@ bool MainWindow::prepareNextMessageFrame()
} else {
ui->nextFreeTextMsg->setText(frame);
/*
int count = m_txFrameCount;
int sent = count - m_txFrameQueue.count();
@ -5425,6 +5431,9 @@ bool MainWindow::prepareNextMessageFrame()
if(count == sent){
m_i3bit |= Varicode::JS8CallLast;
}
*/
m_i3bit = bits;
updateTxButtonDisplay();
@ -8011,8 +8020,8 @@ void MainWindow::refreshTextDisplay(){
QStringList textList;
qDebug() << "frames:";
foreach(auto frame, frames){
auto dt = DecodedText(frame);
foreach(Frame frame, frames){
auto dt = DecodedText(frame.frame, frame.bits);
qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType());
textList.append(dt.message());
}
@ -8053,14 +8062,16 @@ void MainWindow::refreshTextDisplay(){
);
connect(t, &BuildMessageFramesThread::finished, t, &QObject::deleteLater);
connect(t, &BuildMessageFramesThread::resultReady, this, [this, text](const QStringList frames){
connect(t, &BuildMessageFramesThread::resultReady, this, [this, text](QStringList frames, QList<int> bits){
QStringList textList;
qDebug() << "frames:";
int i = 0;
foreach(auto frame, frames){
auto dt = DecodedText(frame);
auto dt = DecodedText(frame, bits.at(i));
qDebug() << "->" << frame << dt.message() << Varicode::frameTypeString(dt.frameType());
textList.append(dt.message());
i++;
}
auto transmitText = textList.join("");
@ -8389,7 +8400,7 @@ void MainWindow::processCompoundActivity() {
bits == Varicode::JS8Call ||
((bits & Varicode::JS8CallFirst) == Varicode::JS8CallFirst) ||
((bits & Varicode::JS8CallLast) == Varicode::JS8CallLast) ||
((bits & Varicode::JS8CallReserved) == Varicode::JS8CallReserved)
((bits & Varicode::JS8CallData) == Varicode::JS8CallData)
);
if (!validBits) {
qDebug() << "-> buffer.cmd bits is invalid...skip";
@ -10346,7 +10357,7 @@ void MainWindow::write_transmit_entry (QString const& file_name)
QTextStream out(&f);
auto time = DriftingDateTime::currentDateTimeUtc ();
time = time.addSecs (-(time.time ().second () % m_TRperiod));
auto dt = DecodedText(m_currentMessage);
auto dt = DecodedText(m_currentMessage, m_currentMessageBits);
out << time.toString("yyyy-MM-dd hh:mm:ss")
<< " Transmitting " << qSetRealNumberPrecision (12) << (m_freqNominal / 1.e6)
<< " MHz " << QString(m_modeTx).replace("FT8", "JS8")

View File

@ -155,7 +155,7 @@ public slots:
void createMessage(QString const& text);
void createMessageTransmitQueue(QString const& text);
void resetMessageTransmitQueue();
QString popMessageFrame();
QPair<QString, int> popMessageFrame();
protected:
void keyPressEvent (QKeyEvent *) override;
void closeEvent(QCloseEvent *) override;
@ -298,7 +298,7 @@ private slots:
void on_nextFreeTextMsg_currentTextChanged (QString const&);
void on_extFreeTextMsgEdit_currentTextChanged (QString const&);
int currentFreqOffset();
QStringList buildMessageFrames(QString const& text);
QList<QPair<QString, int>> buildMessageFrames(QString const& text);
bool prepareNextMessageFrame();
bool isFreqOffsetFree(int f, int bw);
int findFreeFreqOffset(int fmin, int fmax, int bw);
@ -415,7 +415,7 @@ private:
private:
void astroUpdate ();
void writeAllTxt(QString message);
void writeAllTxt(QString message, int bits);
void hideMenus(bool b);
NetworkAccessManager m_network_manager;
@ -553,6 +553,7 @@ private:
bool m_sentFirst73;
int m_currentMessageType;
QString m_currentMessage;
int m_currentMessageBits;
int m_lastMessageType;
QString m_lastMessageSent;
bool m_bShMsgs;
@ -773,7 +774,7 @@ private:
QMap<QString, QVariant> m_showColumnsCache; // table column:key -> show boolean
QMap<QString, QVariant> m_sortCache; // table key -> sort by
QPriorityQueue<PrioritizedMessage> m_txMessageQueue; // messages to be sent
QQueue<QString> m_txFrameQueue; // frames to be sent
QQueue<QPair<QString, int>> m_txFrameQueue; // frames to be sent
QQueue<ActivityDetail> m_rxActivityQueue; // all rx activity queue
QQueue<CommandDetail> m_rxCommandQueue; // command queue for processing commands
QQueue<CallDetail> m_rxCallQueue; // call detail queue for spots to pskreporter

View File

@ -104,7 +104,7 @@ QString callsign_pattern = QString("(?<callsign>[@]?[A-Z0-9/]+)");
QString optional_cmd_pattern = QString("(?<cmd>\\s?(?:HEARTBEAT (ACK|REQ)|AGN[?]|QSL[?]|HW CPY[?]|APRS[:]|QRZ[?]|SNR[?]|QTC[?]|QTH[?]|GRID[?]|STATUS[?]|(?:(?:ACK|73|YES|NO|SNR|QSL|RR|SK|FB|QTH|QTC|GRID|ACTIVE|IDLE)(?=[ ]|$))|[#> ]))?");
QString optional_grid_pattern = QString("(?<grid>\\s?[A-R]{2}[0-9]{2})?");
QString optional_extended_grid_pattern = QString("^(?<grid>\\s?(?:[A-R]{2}[0-9]{2}(?:[A-X]{2}(?:[0-9]{2})?)*))?");
QString optional_num_pattern = QString("(?<num>(?<=SNR|ACK)\\s?[-+]?(?:3[01]|[0-2]?[0-9]))?");
QString optional_num_pattern = QString("(?<num>(?<=SNR|HEARTBEAT ACK)\\s?[-+]?(?:3[01]|[0-2]?[0-9]))?");
QRegularExpression directed_re("^" +
callsign_pattern +
@ -981,7 +981,7 @@ quint8 Varicode::packCmd(quint8 cmd, quint8 num, bool *pPackedNum){
// [1][X][6]
// X = 0 == SNR
// X = 1 == ACK
value = ((1 << 1) | (int)(cmdStr == " ACK")) << 6;
value = ((1 << 1) | (int)(cmdStr == " HEARTBEAT ACK")) << 6;
value = value + (num & ((1<<6)-1));
if(pPackedNum) *pPackedNum = true;
} else {
@ -999,7 +999,7 @@ quint8 Varicode::unpackCmd(quint8 value, quint8 *pNum){
auto cmd = directed_cmds[" SNR"];
if(value & (1<<6)){
cmd = directed_cmds[" ACK"];
cmd = directed_cmds[" HEARTBEAT ACK"];
}
return cmd;
} else {
@ -1459,12 +1459,10 @@ QStringList Varicode::unpackDirectedMessage(const QString &text, quint8 *pType){
QString packHuffMessage(const QString &input, int *n){
static const int frameSize = 72;
QString frame;
QString frame = {false};
// [3][69] = 72
QVector<bool> frameDataBits;
QVector<bool> frameHeaderBits = Varicode::intToBits(Varicode::FrameDataUncompressed, 3);
// [1][71] = 72
QVector<bool> frameBits = {false};
int i = 0;
@ -1474,6 +1472,7 @@ QString packHuffMessage(const QString &input, int *n){
for(it = input.constBegin(); it != input.constEnd(); it++){
auto ch = (*it).toUpper();
if(!validChars.contains(ch)){
if(n) *n = 0;
return frame;
}
}
@ -1482,32 +1481,28 @@ QString packHuffMessage(const QString &input, int *n){
foreach(auto pair, Varicode::huffEncode(Varicode::defaultHuffTable(), input)){
auto charN = pair.first;
auto charBits = pair.second;
if(frameHeaderBits.length() + frameDataBits.length() + charBits.length() < frameSize){
frameDataBits += charBits;
if(frameBits.length() + charBits.length() < frameSize){
frameBits += charBits;
i += charN;
continue;
}
break;
}
QVector<bool> framePadBits;
qDebug() << "Huff bits" << frameBits.length() << "chars" << i;
int pad = frameSize - frameHeaderBits.length() - frameDataBits.length();
int pad = frameSize - frameBits.length();
if(pad){
// the way we will pad is this...
// set the bit after the frame to 0 and every bit after that a 1
// to unpad, seek from the end of the bits until you hit a zero... the rest is the actual frame.
for(int i = 0; i < pad; i++){
framePadBits.append(i == 0 ? (bool)0 : (bool)1);
frameBits.append(i == 0 ? (bool)0 : (bool)1);
}
}
qDebug() << "Huff bits" << frameDataBits.length() << "chars" << i;
QVector<bool> allBits = frameHeaderBits + frameDataBits + framePadBits;
quint64 value = Varicode::bitsToInt(allBits.constBegin(), 64);
quint8 rem = (quint8)Varicode::bitsToInt(allBits.constBegin() + 64, 8);
quint64 value = Varicode::bitsToInt(frameBits.constBegin(), 64);
quint8 rem = (quint8)Varicode::bitsToInt(frameBits.constBegin() + 64, 8);
frame = Varicode::pack72bits(value, rem);
if(n) *n = i;
@ -1520,9 +1515,8 @@ QString packCompressedMessage(const QString &input, int *n){
QString frame;
QVector<bool> frameBits;
frameBits.append(Varicode::intToBits(Varicode::FrameDataCompressed, 3));
// [1][71] = 72
QVector<bool> frameBits = {true};
int i = 0;
foreach(auto pair, JSC::compress(input)){
@ -1538,7 +1532,7 @@ QString packCompressedMessage(const QString &input, int *n){
break;
}
qDebug() << "Compressed bits" << frameBits.length() - 3 << "chars" << i;
qDebug() << "Compressed bits" << frameBits.length() << "chars" << i;
int pad = frameSize - frameBits.length();
if(pad){
@ -1588,25 +1582,25 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){
quint64 value = Varicode::unpack72bits(text, &rem);
auto bits = Varicode::intToBits(value, 64) + Varicode::intToBits(rem, 8);
quint8 type = Varicode::bitsToInt(bits.mid(0, 3));
bool compressed = bits.at(0);
int n = bits.lastIndexOf(0);
bits = bits.mid(3, n-3);
bits = bits.mid(1, n-1);
if(type == FrameDataUncompressed){
if(compressed){
unpacked = JSC::decompress(bits);
if(pType) *pType = Varicode::FrameDataCompressed;
} else {
// huff decode the bits (without escapes)
unpacked = Varicode::huffDecode(Varicode::defaultHuffTable(), bits);
if(pType) *pType = type;
} else if(type == FrameDataCompressed) {
unpacked = JSC::decompress(bits);
if(pType) *pType = type;
if(pType) *pType = Varicode::FrameDataUncompressed;
}
return unpacked;
}
// TODO: remove the dependence on providing all this data?
QStringList Varicode::buildMessageFrames(
QList<QPair<QString, int>> Varicode::buildMessageFrames(
QString const& mycall,
//QString const& basecall,
QString const& mygrid,
@ -1621,7 +1615,7 @@ QStringList Varicode::buildMessageFrames(
bool mycallCompound = Varicode::isCompoundCallsign(mycall);
QStringList frames;
QList<QPair<QString, int>> frames;
foreach(QString line, text.split(QRegExp("[\\r\\n]"), QString::SkipEmptyParts)){
// once we find a directed call, data encode the rest of the line.
@ -1735,13 +1729,13 @@ QStringList Varicode::buildMessageFrames(
}
if(useBcn){
frames.append(frame);
frames.append({ frame, Varicode::JS8Call });
line = line.mid(l);
}
#if ALLOW_SEND_COMPOUND
if(useCmp){
frames.append(frame);
frames.append({ frame, Varicode::JS8Call });
line = line.mid(o);
}
#endif
@ -1775,14 +1769,14 @@ QStringList Varicode::buildMessageFrames(
QString deCompoundMessage = QString("`%1 %2").arg(mycall).arg(mygrid);
QString deCompoundFrame = Varicode::packCompoundMessage(deCompoundMessage, nullptr);
if(!deCompoundFrame.isEmpty()){
frames.append(deCompoundFrame);
frames.append({ deCompoundFrame, Varicode::JS8Call });
}
// Followed, by a standard OR compound directed message...
QString dirCompoundMessage = QString("`%1%2%3").arg(dirTo).arg(dirCmd).arg(dirNum);
QString dirCompoundFrame = Varicode::packCompoundMessage(dirCompoundMessage, nullptr);
if(!dirCompoundFrame.isEmpty()){
frames.append(dirCompoundFrame);
frames.append({ dirCompoundFrame, Varicode::JS8Call });
}
shouldUseStandardFrame = false;
}
@ -1790,7 +1784,7 @@ QStringList Varicode::buildMessageFrames(
if(shouldUseStandardFrame) {
// otherwise, just send the standard directed frame
frames.append(frame);
frames.append({ frame, Varicode::JS8Call });
}
line = line.mid(n);
@ -1818,12 +1812,17 @@ QStringList Varicode::buildMessageFrames(
}
if(useDat){
frames.append(frame);
frames.append({ frame, Varicode::JS8CallData });
line = line.mid(m);
}
}
}
if(!frames.isEmpty()){
frames.first().second |= Varicode::JS8CallFirst;
frames.last().second |= Varicode::JS8CallLast;
}
return frames;
}
@ -1854,5 +1853,12 @@ void BuildMessageFramesThread::run(){
m_text
);
emit resultReady(results);
QList<QString> frames;
QList<int> bits;
foreach(auto pair, results){
frames.append(pair.first);
bits.append(pair.second);
}
emit resultReady(frames, bits);
}

View File

@ -12,6 +12,7 @@
#include <QVector>
#include <QThread>
class Varicode
{
public:
@ -20,12 +21,12 @@ public:
JS8Call = 0, // [000] <- any other frame of the message
JS8CallFirst = 1, // [001] <- the first frame of a message
JS8CallLast = 2, // [010] <- the last frame of a message
JS8CallReserved = 4, // [100] <- a reserved flag for future use...
JS8CallData = 4, // [100] <- raw data frame (no frame type header)
};
enum FrameType {
FrameUnknown = 255, // [11111111] <- only used as a sentinel
FrameHeartbeat = 0, // [000]
FrameHeartbeat = 0, // [000]
FrameCompound = 1, // [001]
FrameCompoundDirected = 2, // [010]
FrameDirected = 3, // [011]
@ -147,7 +148,7 @@ public:
static QString packDataMessage(QString const& text, int *n);
static QString unpackDataMessage(QString const& text, quint8 *pType);
static QStringList buildMessageFrames(
static QList<QPair<QString, int>> buildMessageFrames(
QString const& mycall,
//QString const& basecall,
QString const& mygrid,
@ -171,7 +172,7 @@ public:
QObject *parent=nullptr);
void run() override;
signals:
void resultReady(const QStringList s);
void resultReady(QStringList, QList<int>);
private:
QString m_mycall;