chris/js8call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Working through compression implementation for data frames

Browse Source
This commit is contained in:
Jordan Sherer 2018-10-01 09:57:37 -04:00
parent 489cf3a85c
commit 2f6ed1b89c
5 changed files with 121 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
jsc.cpp
View File

@ -24,10 +24,10 @@
#include <cmath>
Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c){
Codeword JSC::codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c){
QList<Codeword> out;
quint32 v = ((index % s) << 1) + (int)separate;
quint32 v = ((index % s) << 1) + (quint32)separate;
out.prepend(Varicode::intToBits(v, bytesize + 1));
quint32 x = index / s;
@ -45,78 +45,41 @@ Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, qui
return word;
}
QPair<CompressionMap, CompressionList> JSC::loadCompressionTable(){
CompressionMap out;
CompressionList words;
for(int i = 0; i < JSC::size; i++){
out[JSC::map[i]] = i;
}
words.reserve(JSC::size);
for(int i = 0; i < JSC::size; i++){
words.append(JSC::list[i]);
}
return {out, words};
}
QPair<CompressionMap, CompressionList> JSC::loadCompressionTable(QTextStream &stream){
CompressionMap out;
CompressionList words;
int index = 0;
while(!stream.atEnd()){
// assume that this is in sorted order, that each word is already upper case.
auto word = stream.readLine().trimmed();
if(out.contains(word)){
continue;
}
out[word] = index;
words.append(word);
index++;
}
#if 1
qStableSort(words.begin(), words.end(), [out](QString const &left, QString const &right){
return out[left] < out[right];
});
qStableSort(words.begin(), words.end(), [](QString const &left, QString const &right){
if(left.length() == right.length()){
return left < right;
}
return left.length() < right.length();
});
#endif
return {out, words};
}
QList<CodewordPair> JSC::compress(CompressionTable table, QString text){
QList<CodewordPair> JSC::compress(QString text){
QList<CodewordPair> out;
const quint32 b = 4;
const quint32 s = 7;
const quint32 c = pow(2, 4) - s;
auto map = table.first;
auto list = table.second;
foreach(QString w, text.split(" ", QString::SkipEmptyParts)){
if(map.contains(w)){
auto index = map[w];
out.append({ codeword(index, true, b, s, c), w.length() });
bool ok = false;
auto index = lookup(w, &ok);
if(ok){
// cool, we found the word...
out.append({ codeword(index, true, b, s, c), (quint32)w.length() + 1 /* for the space that follows */ });
} else {
// prefix match?
// hmm. no dice. let's go for a prefix match
while(!w.isEmpty()){
bool hasPrefix = false;
auto d = w.toLatin1().data();
for(quint32 i = 0; i < JSC::size; i++){
// TODO: we could probably precompute these sizes
quint32 len = strlen(JSC::list[i]);
if(strncmp(d, JSC::list[i], len) == 0){
w = QString(w.mid(len));
auto word = JSC::list[i];
auto index = lookup(word, &ok);
if(ok){
bool isLast = w.isEmpty();
out.append({ codeword(index, isLast, b, s, c), len + (isLast ? 1 : 0) /* for the space that follows */});
hasPrefix = true;
break;
}
foreach(QString word, list){
if(w.startsWith(word)){
w = QString(w.mid(word.length()));
out.append({ codeword(map[word], w.isEmpty(), b, s, c), word.length()});
hasPrefix = true;
break;
}
}
@ -132,7 +95,7 @@ QList<CodewordPair> JSC::compress(CompressionTable table, QString text){
return out;
}
QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
QString JSC::decompress(Codeword const& bitvec){
const quint32 b = 4;
const quint32 s = 7;
const quint32 c = pow(2, b) - s;
@ -177,7 +140,7 @@ QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
j = j*s + bytes[start + k] + base[k];
out.append(table.first.key(j));
out.append(QString(JSC::map[j])); // table.first.key(j));
if(separators.first() == start + k){
out.append(" ");
separators.removeFirst();
@ -188,3 +151,19 @@ QString JSC::decompress(CompressionTable table, Codeword const& bitvec){
return out.join("");
}
quint32 JSC::lookup(QString w, bool * ok){
return lookup(w.toLatin1().data(), ok);
}
quint32 JSC::lookup(char const* b, bool *ok){
for(quint32 i = 0; i < JSC::size; i++){
if(strcmp(b, JSC::map[i]) == 0){
if(ok) *ok = true;
return i;
}
}
if(ok) *ok = false;
return 0;
}

20
jsc.h
View File

@ -12,21 +12,25 @@
#include <QPair>
#include <QVector>
typedef QMap<QString, int> CompressionMap; // Map(Word, I) where I is the word index sorted by frequency
typedef QList<QString> CompressionList; // List(Word) where list is sorted
typedef QPair<CompressionMap, CompressionList> CompressionTable; // Tuple(Map, List)
typedef QPair<QVector<bool>, int> CodewordPair; // Tuple(Codeword, N) where N = number of characters
typedef QVector<bool> Codeword;
typedef QPair<QVector<bool>, quint32> CodewordPair; // Tuple(Codeword, N) where N = number of characters
typedef QVector<bool> Codeword; // Codeword bit vector
class JSC
{
public:
#if 0
static CompressionTable loadCompressionTable();
static CompressionTable loadCompressionTable(QTextStream &stream);
static QList<CodewordPair> compress(CompressionTable table, QString text);
static QString decompress(CompressionTable table, Codeword const& bits);
#endif
static Codeword codeword(quint32 index, bool separate, quint32 bytesize, quint32 s, quint32 c);
static QList<CodewordPair> compress(QString text);
static QString decompress(Codeword const& bits);
static const int size = 233638;
static quint32 lookup(QString w, bool *ok);
static quint32 lookup(char const* b, bool *ok);
static const quint32 size = 233638;
static char const* map[];
static char const* list[];
};

45
mainwindow.cpp
View File

@ -1450,18 +1450,6 @@ void MainWindow::not_GA_warning_message ()
}
void MainWindow::initializeDummyData(){
auto table = JSC::loadCompressionTable();
Codeword bits;
auto compressed = JSC::compress(table, "E A T EAT TEA ATE EATTET");
foreach(auto pair, compressed){
qDebug() << "compressed" << Varicode::bitsToStr(pair.first);
bits.append(pair.first);
}
qDebug() << "decomressed" << JSC::decompress(table, bits);
if(!QApplication::applicationName().contains("dummy")){
return;
}
@ -2773,6 +2761,11 @@ void MainWindow::createStatusBar() //createStatusBar
progressBar.setMinimumSize (QSize {100, 18});
progressBar.setFormat ("%v/%m");
statusBar()->addPermanentWidget(&wpm_label);
wpm_label.setMinimumSize (QSize {90, 18});
wpm_label.setFrameStyle (QFrame::Panel | QFrame::Sunken);
wpm_label.setAlignment(Qt::AlignHCenter);
statusBar ()->addPermanentWidget (&watchdog_label);
update_watchdog_label ();
}
@ -6134,15 +6127,6 @@ void MainWindow::on_extFreeTextMsgEdit_currentTextChanged (QString const& text)
ui->extFreeTextMsgEdit->setTextCursor(c);
}
int count = countFT8MessageFrames(x);
if(count > 0){
ui->startTxButton->setText(QString("Send (%1)").arg(count));
ui->startTxButton->setEnabled(true);
} else {
ui->startTxButton->setText("Send");
ui->startTxButton->setEnabled(false);
}
}
int MainWindow::currentFreqOffset(){
@ -9009,6 +8993,25 @@ void MainWindow::updateButtonDisplay(){
int count = m_txFrameCount;
int sent = count - m_txFrameQueue.count();
ui->startTxButton->setText(m_tune ? "Tuning" : QString("Sending (%1/%2)").arg(sent).arg(count));
} else {
// TODO: only if text changed
auto text = ui->extFreeTextMsgEdit->toPlainText();
int count = countFT8MessageFrames(text);
if(count > 0){
ui->startTxButton->setText(QString("Send (%1)").arg(count));
ui->startTxButton->setEnabled(true);
auto words = text.split(" ", QString::SkipEmptyParts).length();
auto wpm = QString::number(words/(count/4.0), 'g', 2);
wpm_label.setText(QString("%1 wpm").arg(wpm));
} else {
ui->startTxButton->setText("Send");
ui->startTxButton->setEnabled(false);
wpm_label.clear();
}
}
}

2
mainwindow.h
View File

@ -620,6 +620,7 @@ private:
QLabel band_hopping_label;
QProgressBar progressBar;
QLabel watchdog_label;
QLabel wpm_label;
QFuture<void> m_wav_future;
QFutureWatcher<void> m_wav_future_watcher;
@ -934,6 +935,7 @@ private:
void write_transmit_entry (QString const& file_name);
};
extern int killbyname(const char* progName);
extern void getDev(int* numDevices,char hostAPI_DeviceName[][50],
int minChan[], int maxChan[],

39
varicode.cpp
View File

@ -26,6 +26,7 @@
#include "crc.h"
#include "varicode.h"
#include "jsc.h"
const int nalphabet = 41;
QString alphabet = {"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ+-./?"}; // alphabet to encode _into_ for FT8 freetext transmission
@ -1527,7 +1528,42 @@ QString Varicode::packDataMessage(const QString &input, int *n){
static const int frameSize = 72;
QString frame;
QVector<bool> frameBits;
frameBits.append(Varicode::intToBits(FrameDataPadded, 3));
int i = 0;
foreach(auto pair, JSC::compress(input)){
auto bits = pair.first;
auto chars = pair.second;
if(frameBits.length() + bits.length() < frameSize){
frameBits.append(bits);
i += chars;
continue;
}
break;
}
int pad = frameSize - frameBits.length();
if(pad){
// the way we will pad is this...
// set the bit after the frame to 0 and every bit after that a 1
// to unpad, seek from the end of the bits until you hit a zero... the rest is the actual frame.
for(int i = 0; i < pad; i++){
frameBits.append(i == 0 ? (bool)0 : (bool)1);
}
}
quint64 value = Varicode::bitsToInt(frameBits.constBegin(), 64);
quint8 rem = (quint8)Varicode::bitsToInt(frameBits.constBegin() + 64, 8);
frame = Varicode::pack72bits(value, rem);
*n = i;
#if 0
// [3][69] = 72
QVector<bool> frameDataBits;
@ -1568,6 +1604,7 @@ QString Varicode::packDataMessage(const QString &input, int *n){
frame = Varicode::pack72bits(value, rem);
*n = i;
#endif
return frame;
}
@ -1583,6 +1620,7 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){
quint64 value = Varicode::unpack72bits(text, &rem);
auto bits = Varicode::intToBits(value, 64) + Varicode::intToBits(rem, 8);
#if 0
quint8 type = Varicode::bitsToInt(bits.mid(0, 3));
if(type == FrameDataUnpadded){
bits = bits.mid(3);
@ -1600,6 +1638,7 @@ QString Varicode::unpackDataMessage(const QString &text, quint8 *pType){
unpacked = Varicode::huffUnescape(unpacked);
if(pType) *pType = type;
#endif
return unpacked;
}