Merged master 8748

This commit is contained in:
Jordan Sherer
2018-08-05 11:33:30 -04:00
parent 8f8772f1bd
commit 62899069bf
1095 changed files with 31298 additions and 367679 deletions
File diff suppressed because it is too large Load Diff
@@ -1,238 +0,0 @@
program ldpcsim174
! End to end test of the (174,75)/crc12 encoder and decoder.
use crc
use packjt
parameter(NRECENT=10)
character*12 recent_calls(NRECENT)
character*22 msg,msgsent,msgreceived
character*8 arg
integer*1, allocatable :: codeword(:), decoded(:), message(:)
integer*1, target:: i1Msg8BitBytes(11)
integer*1 msgbits(87)
integer*1 apmask(174), cw(174)
integer*2 checksum
integer*4 i4Msg6BitWords(13)
integer colorder(174)
integer nerrtot(174),nerrdec(174),nmpcbad(87)
logical checksumok,fsk,bpsk
real*8, allocatable :: rxdata(:)
real, allocatable :: llr(:)
data colorder/ &
0, 1, 2, 3, 30, 4, 5, 6, 7, 8, 9, 10, 11, 32, 12, 40, 13, 14, 15, 16,&
17, 18, 37, 45, 29, 19, 20, 21, 41, 22, 42, 31, 33, 34, 44, 35, 47, 51, 50, 43,&
36, 52, 63, 46, 25, 55, 27, 24, 23, 53, 39, 49, 59, 38, 48, 61, 60, 57, 28, 62,&
56, 58, 65, 66, 26, 70, 64, 69, 68, 67, 74, 71, 54, 76, 72, 75, 78, 77, 80, 79,&
73, 83, 84, 81, 82, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,&
100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,&
120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,&
140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,&
160,161,162,163,164,165,166,167,168,169,170,171,172,173/
do i=1,NRECENT
recent_calls(i)=' '
enddo
nerrtot=0
nerrdec=0
nmpcbad=0 ! Used to collect the number of errors in the message+crc part of the codeword
nargs=iargc()
if(nargs.ne.4) then
print*,'Usage: ldpcsim niter ndepth #trials s '
print*,'eg: ldpcsim 10 2 1000 0.84'
print*,'belief propagation iterations: niter, ordered-statistics depth: ndepth'
print*,'If s is negative, then value is ignored and sigma is calculated from SNR.'
return
endif
call getarg(1,arg)
read(arg,*) max_iterations
call getarg(2,arg)
read(arg,*) ndepth
call getarg(3,arg)
read(arg,*) ntrials
call getarg(4,arg)
read(arg,*) s
fsk=.false.
bpsk=.true.
! don't count crc bits as data bits
N=174
K=87
! scale Eb/No for a (174,87) code
rate=real(K)/real(N)
write(*,*) "rate: ",rate
write(*,*) "niter= ",max_iterations," s= ",s
allocate ( codeword(N), decoded(K), message(K) )
allocate ( rxdata(N), llr(N) )
msg="K1JT K9AN EN50"
! msg="G4WJS K9AN EN50"
call packmsg(msg,i4Msg6BitWords,itype,.false.) !Pack into 12 6-bit bytes
call unpackmsg(i4Msg6BitWords,msgsent,.false.,'') !Unpack to get msgsent
write(*,*) "message sent ",msgsent
i4=0
ik=0
im=0
do i=1,12
nn=i4Msg6BitWords(i)
do j=1, 6
ik=ik+1
i4=i4+i4+iand(1,ishft(nn,j-6))
i4=iand(i4,255)
if(ik.eq.8) then
im=im+1
! if(i4.gt.127) i4=i4-256
i1Msg8BitBytes(im)=i4
ik=0
endif
enddo
enddo
i1Msg8BitBytes(10:11)=0
checksum = crc12 (c_loc (i1Msg8BitBytes), 11)
! For reference, the next 3 lines show how to check the CRC
i1Msg8BitBytes(10)=checksum/256
i1Msg8BitBytes(11)=iand (checksum,255)
checksumok = crc12_check(c_loc (i1Msg8BitBytes), 11)
if( checksumok ) write(*,*) 'Good checksum'
! K=87, For now:
! msgbits(1:72) JT message bits
! msgbits(73:75) 3 free message bits (set to 0)
! msgbits(76:87) CRC12
mbit=0
do i=1, 9
i1=i1Msg8BitBytes(i)
do ibit=1,8
mbit=mbit+1
msgbits(mbit)=iand(1,ishft(i1,ibit-8))
enddo
enddo
msgbits(73:75)=0 ! the three extra message bits go here
i1=i1Msg8BitBytes(10) ! First 4 bits of crc12 are LSB of this byte
do ibit=1,4
msgbits(75+ibit)=iand(1,ishft(i1,ibit-4))
enddo
i1=i1Msg8BitBytes(11) ! Now shift in last 8 bits of the CRC
do ibit=1,8
msgbits(79+ibit)=iand(1,ishft(i1,ibit-8))
enddo
write(*,*) 'message'
write(*,'(11(8i1,1x))') msgbits
call encode174(msgbits,codeword)
call init_random_seed()
! call sgran()
write(*,*) 'codeword'
write(*,'(22(8i1,1x))') codeword
write(*,*) "Es/N0 SNR2500 ngood nundetected nbadcrc sigma"
do idb = 20,-10,-1
!do idb = -3,-3,-1
db=idb/2.0-1.0
sigma=1/sqrt( 2*(10**(db/10.0)) )
ngood=0
nue=0
nbadcrc=0
nberr=0
do itrial=1, ntrials
! Create a realization of a noisy received word
do i=1,N
if( bpsk ) then
rxdata(i) = 2.0*codeword(i)-1.0 + sigma*gran()
elseif( fsk ) then
if( codeword(i) .eq. 1 ) then
r1=(1.0 + sigma*gran())**2 + (sigma*gran())**2
r2=(sigma*gran())**2 + (sigma*gran())**2
elseif( codeword(i) .eq. 0 ) then
r2=(1.0 + sigma*gran())**2 + (sigma*gran())**2
r1=(sigma*gran())**2 + (sigma*gran())**2
endif
! rxdata(i)=0.35*(sqrt(r1)-sqrt(r2))
! rxdata(i)=0.35*(exp(r1)-exp(r2))
rxdata(i)=0.12*(log(r1)-log(r2))
endif
enddo
nerr=0
do i=1,N
if( rxdata(i)*(2*codeword(i)-1.0) .lt. 0 ) nerr=nerr+1
enddo
nerrtot(nerr)=nerrtot(nerr)+1
nberr=nberr+nerr
! Correct signal normalization is important for this decoder.
rxav=sum(rxdata)/N
rx2av=sum(rxdata*rxdata)/N
rxsig=sqrt(rx2av-rxav*rxav)
rxdata=rxdata/rxsig
! To match the metric to the channel, s should be set to the noise standard deviation.
! For now, set s to the value that optimizes decode probability near threshold.
! The s parameter can be tuned to trade a few tenth's dB of threshold for an order of
! magnitude in UER
if( s .lt. 0 ) then
ss=sigma
else
ss=s
endif
llr=2.0*rxdata/(ss*ss)
nap=0 ! number of AP bits
llr(colorder(174-87+1:174-87+nap)+1)=5*(2.0*msgbits(1:nap)-1.0)
apmask=0
apmask(colorder(174-87+1:174-87+nap)+1)=1
! max_iterations is max number of belief propagation iterations
call bpdecode174(llr, apmask, max_iterations, decoded, cw, nharderrors,niterations)
if( ndepth .ge. 0 .and. nharderrors .lt. 0 ) call osd174(llr, apmask, ndepth, decoded, cw, nharderrors, dmin)
! If the decoder finds a valid codeword, nharderrors will be .ge. 0.
if( nharderrors .ge. 0 ) then
call extractmessage174(decoded,msgreceived,ncrcflag,recent_calls,nrecent)
if( ncrcflag .ne. 1 ) then
nbadcrc=nbadcrc+1
endif
nueflag=0
nerrmpc=0
do i=1,K ! find number of errors in message+crc part of codeword
if( msgbits(i) .ne. decoded(i) ) then
nueflag=1
nerrmpc=nerrmpc+1
endif
enddo
nmpcbad(nerrmpc)=nmpcbad(nerrmpc)+1
if( ncrcflag .eq. 1 ) then
if( nueflag .eq. 0 ) then
ngood=ngood+1
nerrdec(nerr)=nerrdec(nerr)+1
else if( nueflag .eq. 1 ) then
nue=nue+1;
endif
endif
endif
enddo
baud=12000/1920
snr2500=db+10.0*log10((baud/2500.0))
pberr=real(nberr)/(real(ntrials*N))
write(*,"(f4.1,4x,f5.1,1x,i8,1x,i8,1x,i8,8x,f5.2,8x,e10.3)") db,snr2500,ngood,nue,nbadcrc,ss,pberr
enddo
open(unit=23,file='nerrhisto.dat',status='unknown')
do i=1,174
write(23,'(i4,2x,i10,i10,f10.2)') i,nerrdec(i),nerrtot(i),real(nerrdec(i))/real(nerrtot(i)+1e-10)
enddo
close(23)
open(unit=25,file='nmpcbad.dat',status='unknown')
do i=1,87
write(25,'(i4,2x,i10)') i,nmpcbad(i)
enddo
close(25)
end program ldpcsim174
@@ -1,418 +0,0 @@
#ifndef NETWORK_MESSAGE_HPP__
#define NETWORK_MESSAGE_HPP__
/*
* WSJT-X Message Formats
* ======================
*
* All messages are written or read using the QDataStream derivatives
* defined below, note that we are using the default for floating
* point precision which means all are double precision i.e. 64-bit
* IEEE format.
*
* Message is big endian format
*
* Header format:
*
* 32-bit unsigned integer magic number 0xadbccbda
* 32-bit unsigned integer schema number
*
* Payload format:
*
* As per the QDataStream format, see below for version used and
* here:
*
* http://doc.qt.io/qt-5/datastreamformat.html
*
* for the serialization details for each type, at the time of
* writing the above document is for Qt_5_0 format which is buggy
* so we use Qt_5_4 format, differences are:
*
* QDateTime:
* QDate qint64 Julian day number
* QTime quint32 Milli-seconds since midnight
* timespec quint8 0=local, 1=UTC, 2=Offset from UTC
* (seconds)
* 3=time zone
* offset qint32 only present if timespec=2
* timezone several-fields only present if timespec=3
*
* we will avoid using QDateTime fields with time zones for simplicity.
*
* Type utf8 is a utf-8 byte string formatted as a QByteArray for
* serialization purposes (currently a quint32 size followed by size
* bytes, no terminator is present or counted).
*
* The QDataStream format document linked above is not complete for
* the QByteArray serialization format, it is similar to the QString
* serialization format in that it differentiates between empty
* strings and null strings. Empty strings have a length of zero
* whereas null strings have a length field of 0xffffffff.
*
* Schema Negotiation
* ------------------
*
* The NetworkMessage::Builder class specifies a schema number which
* may be incremented from time to time. It represents a version of
* the underlying encoding schemes used to store data items. Since the
* underlying encoding is defined by the Qt project in it's
* QDataStream stream operators, it is essential that clients and
* servers of this protocol can agree on a common scheme. The
* NetworkMessage utility classes below exchange the schema number
* actually used. The handling of the schema is backwards compatible
* to an extent, so long as clients and servers are written
* correctly. For example a server written to any particular schema
* version can communicate with a client written to a later schema.
*
* Schema Version 1:- this schema used the QDataStream::Qt_5_0 version
* which is broken.
*
* Schema Version 2:- this schema uses the QDataStream::Qt_5_2 version.
*
* Schema Version 3:- this schema uses the QDataStream::Qt_5_4 version.
*
*
*
* Message Direction Value Type
* ------------- --------- ---------------------- -----------
* Heartbeat Out/In 0 quint32
* Id (unique key) utf8
* Maximum schema number quint32
* version utf8
* revision utf8
*
* The heartbeat message shall be sent on a periodic basis every
* NetworkMessage::pulse seconds (see below), the WSJT-X
* application does that using the MessageClient class. This
* message is intended to be used by servers to detect the presence
* of a client and also the unexpected disappearance of a client
* and by clients to learn the schema negotiated by the server
* after it receives the initial heartbeat message from a client.
* The message_aggregator reference server does just that using the
* MessageServer class. Upon initial startup a client must send a
* heartbeat message as soon as is practical, this message is used
* to negotiate the maximum schema number common to the client and
* server. Note that the server may not be able to support the
* client's requested maximum schema number, in which case the
* first message received from the server will specify a lower
* schema number (never a higher one as that is not allowed). If a
* server replies with a lower schema number then no higher than
* that number shall be used for all further outgoing messages from
* either clients or the server itself.
*
* Note: the "Maximum schema number" field was introduced at the
* same time as schema 3, therefore servers and clients must assume
* schema 2 is the highest schema number supported if the Heartbeat
* message does not contain the "Maximum schema number" field.
*
*
* Status Out 1 quint32
* Id (unique key) utf8
* Dial Frequency (Hz) quint64
* Mode utf8
* DX call utf8
* Report utf8
* Tx Mode utf8
* Tx Enabled bool
* Transmitting bool
* Decoding bool
* Rx DF qint32
* Tx DF qint32
* DE call utf8
* DE grid utf8
* DX grid utf8
* Tx Watchdog bool
* Sub-mode utf8
* Fast mode bool
*
* WSJT-X sends this status message when various internal state
* changes to allow the server to track the relevant state of each
* client without the need for polling commands. The current state
* changes that generate status messages are:
*
* Application start up,
* "Enable Tx" button status changes,
* Dial frequency changes,
* Changes to the "DX Call" field,
* Operating mode, sub-mode or fast mode changes,
* Transmit mode changed (in dual JT9+JT65 mode),
* Changes to the "Rpt" spinner,
* After an old decodes replay sequence (see Replay below),
* When switching between Tx and Rx mode,
* At the start and end of decoding,
* When the Rx DF changes,
* When the Tx DF changes,
* When the DE call or grid changes (currently when settings are exited),
* When the DX call or grid changes,
* When the Tx watchdog is set or reset.
*
*
* Decode Out 2 quint32
* Id (unique key) utf8
* New bool
* Time QTime
* snr qint32
* Delta time (S) float (serialized as double)
* Delta frequency (Hz) quint32
* Mode utf8
* Message utf8
* Low confidence bool
* Off air bool
*
* The decode message is sent when a new decode is completed, in
* this case the 'New' field is true. It is also used in response
* to a "Replay" message where each old decode in the "Band
* activity" window, that has not been erased, is sent in order
* as a one of these messages with the 'New' field set to false.
* See the "Replay" message below for details of usage. Low
* confidence decodes are flagged in protocols where the decoder
* has knows that a decode has a higher than normal probability
* of being false, they should not be reported on publicly
* accessible services without some attached warning or further
* validation. Off air decodes are those that result from playing
* back a .WAV file.
*
*
* Clear Out 3 quint32
* Id (unique key) utf8
*
* This message is send when all prior "Decode" messages in the
* "Band activity" window have been discarded and therefore are
* no long available for actioning with a "Reply" message. It is
* sent when the user erases the "Band activity" window and when
* WSJT-X closes down normally. The server should discard all
* decode messages upon receipt of this message.
*
*
* Reply In 4 quint32
* Id (target unique key) utf8
* Time QTime
* snr qint32
* Delta time (S) float (serialized as double)
* Delta frequency (Hz) quint32
* Mode utf8
* Message utf8
* Low confidence bool
* Modifiers quint8
*
* In order for a server to provide a useful cooperative service
* to WSJT-X it is possible for it to initiate a QSO by sending
* this message to a client. WSJT-X filters this message and only
* acts upon it if the message exactly describes a prior decode
* and that decode is a CQ or QRZ message. The action taken is
* exactly equivalent to the user double clicking the message in
* the "Band activity" window. The intent of this message is for
* servers to be able to provide an advanced look up of potential
* QSO partners, for example determining if they have been worked
* before or if working them may advance some objective like
* award progress. The intention is not to provide a secondary
* user interface for WSJT-X, it is expected that after QSO
* initiation the rest of the QSO is carried out manually using
* the normal WSJT-X user interface.
*
* The Modifiers field allows the equivalent of keyboard
* modifiers to be sent "as if" those modifier keys where pressed
* while double-clicking the specified decoded message. The
* modifier values (hexadecimal) are as follows:
*
* no modifier 0x00
* SHIFT 0x02
* CTRL 0x04 CMD on Mac
* ALT 0x08
* META 0x10 Windows key on MS Windows
* KEYPAD 0x20 Keypad or arrows
* Group switch 0x40 X11 only
*
*
* QSO Logged Out 5 quint32
* Id (unique key) utf8
* Date & Time Off QDateTime
* DX call utf8
* DX grid utf8
* Dial frequency (Hz) quint64
* Mode utf8
* Report send utf8
* Report received utf8
* Tx power utf8
* Comments utf8
* Name utf8
* Date & Time On QDateTime
*
* The QSO logged message is sent to the server(s) when the
* WSJT-X user accepts the "Log QSO" dialog by clicking the "OK"
* button.
*
*
* Close Out 6 quint32
* Id (unique key) utf8
*
* Close is sent by a client immediately prior to it shutting
* down gracefully.
*
*
* Replay In 7 quint32
* Id (unique key) utf8
*
* When a server starts it may be useful for it to determine the
* state of preexisting clients. Sending this message to each
* client as it is discovered will cause that client (WSJT-X) to
* send a "Decode" message for each decode currently in its "Band
* activity" window. Each "Decode" message sent will have the
* "New" flag set to false so that they can be distinguished from
* new decodes. After all the old decodes have been broadcast a
* "Status" message is also broadcast. If the server wishes to
* determine the status of a newly discovered client; this
* message should be used.
*
*
* Halt Tx In 8
* Id (unique key) utf8
* Auto Tx Only bool
*
* The server may stop a client from transmitting messages either
* immediately or at the end of the current transmission period
* using this message.
*
*
* Free Text In 9
* Id (unique key) utf8
* Text utf8
* Send bool
*
* This message allows the server to set the current free text
* message content. Sending this message with a non-empty "Text"
* field is equivalent to typing a new message (old contents are
* discarded) in to the WSJT-X free text message field or "Tx5"
* field (both are updated) and if the "Send" flag is set then
* clicking the "Now" radio button for the "Tx5" field if tab one
* is current or clicking the "Free msg" radio button if tab two
* is current.
*
* It is the responsibility of the sender to limit the length of
* the message text and to limit it to legal message
* characters. Despite this, it may be difficult for the sender
* to determine the maximum message length without reimplementing
* the complete message encoding protocol. Because of this is may
* be better to allow any reasonable message length and to let
* the WSJT-X application encode and possibly truncate the actual
* on-air message.
*
* If the message text is empty the meaning of the message is
* refined to send the current free text unchanged when the
* "Send" flag is set or to clear the current free text when the
* "Send" flag is unset. Note that this API does not include a
* command to determine the contents of the current free text
* message.
*
* WSPRDecode Out 10 quint32
* Id (unique key) utf8
* New bool
* Time QTime
* snr qint32
* Delta time (S) float (serialized as double)
* Frequency (Hz) quint64
* Drift (Hz) qint32
* Callsign utf8
* Grid utf8
* Power (dBm) qint32
* Off air bool
*
* The decode message is sent when a new decode is completed, in
* this case the 'New' field is true. It is also used in response
* to a "Replay" message where each old decode in the "Band
* activity" window, that has not been erased, is sent in order
* as a one of these messages with the 'New' field set to
* false. See the "Replay" message below for details of
* usage. The off air field indicates that the decode was decoded
* from a played back recording.
*
*
*/
#include <QDataStream>
#include "pimpl_h.hpp"
class QIODevice;
class QByteArray;
class QString;
namespace NetworkMessage
{
// NEVER DELETE MESSAGE TYPES
enum Type
{
Heartbeat,
Status,
Decode,
Clear,
Reply,
QSOLogged,
Close,
Replay,
HaltTx,
FreeText,
WSPRDecode,
maximum_message_type_ // ONLY add new message types
// immediately before here
};
quint32 constexpr pulse {15}; // seconds
//
// NetworkMessage::Builder - build a message containing serialized Qt types
//
class Builder
: public QDataStream
{
public:
static quint32 constexpr magic {0xadbccbda}; // never change this
// increment this if a newer Qt schema is required and add decode
// logic to the Builder and Reader class implementations
#if QT_VERSION >= 0x050400
static quint32 constexpr schema_number {3};
#elif QT_VERSION >= 0x050200
static quint32 constexpr schema_number {2};
#else
// Schema 1 (Qt_5_0) is broken
#error "Qt version 5.2 or greater required"
#endif
explicit Builder (QIODevice *, Type, QString const& id, quint32 schema);
explicit Builder (QByteArray *, Type, QString const& id, quint32 schema);
Builder (Builder const&) = delete;
Builder& operator = (Builder const&) = delete;
private:
void common_initialization (Type type, QString const& id, quint32 schema);
};
//
// NetworkMessage::Reader - read a message containing serialized Qt types
//
// Message is as per NetworkMessage::Builder above, the schema()
// member may be used to determine the schema of the original
// message.
//
class Reader
: public QDataStream
{
public:
explicit Reader (QIODevice *);
explicit Reader (QByteArray const&);
Reader (Reader const&) = delete;
Reader& operator = (Reader const&) = delete;
~Reader ();
quint32 schema () const;
Type type () const;
QString id () const;
private:
class impl;
pimpl<impl> m_;
};
}
#endif