Finish implementing legacy protocol, start implementing new protocol

Makefile.in

@@ -820,9 +820,9 @@ am__libtgvoip_la_SOURCES_DIST = TgVoip.cpp VoIPController.cpp \
 	controller/net/PacketReassembler.cpp \
 	controller/protocol/PacketManager.cpp \
 	controller/protocol/PacketStructs.cpp \
-	controller/protocol/packets/Interface.cpp \
+	controller/protocol/packets/Extra.cpp VoIPGroupController.cpp \
-	VoIPGroupController.cpp VoIPServerConfig.cpp audio/AudioIO.cpp \
+	VoIPServerConfig.cpp audio/AudioIO.cpp audio/AudioInput.cpp \
-	audio/AudioInput.cpp audio/AudioOutput.cpp audio/Resampler.cpp \
+	audio/AudioOutput.cpp audio/Resampler.cpp \
 	audio/AudioInputTester.cpp os/posix/NetworkSocketPosix.cpp \
 	video/VideoSource.cpp video/VideoRenderer.cpp \
 	video/VideoPacketSender.cpp video/VideoFEC.cpp \
@@ -1763,9 +1763,9 @@ am__objects_12 = TgVoip.lo VoIPController.lo tools/Buffers.lo \
 	controller/net/PacketReassembler.lo \
 	controller/protocol/PacketManager.lo \
 	controller/protocol/PacketStructs.lo \
-	controller/protocol/packets/Interface.lo \
+	controller/protocol/packets/Extra.lo VoIPGroupController.lo \
-	VoIPGroupController.lo VoIPServerConfig.lo audio/AudioIO.lo \
+	VoIPServerConfig.lo audio/AudioIO.lo audio/AudioInput.lo \
-	audio/AudioInput.lo audio/AudioOutput.lo audio/Resampler.lo \
+	audio/AudioOutput.lo audio/Resampler.lo \
 	audio/AudioInputTester.lo os/posix/NetworkSocketPosix.lo \
 	video/VideoSource.lo video/VideoRenderer.lo \
 	video/VideoPacketSender.lo video/VideoFEC.lo \
@@ -2081,7 +2081,7 @@ am__depfiles_remade = ./$(DEPDIR)/TgVoip.Plo \
 	controller/net/$(DEPDIR)/PacketReassembler.Plo \
 	controller/protocol/$(DEPDIR)/PacketManager.Plo \
 	controller/protocol/$(DEPDIR)/PacketStructs.Plo \
-	controller/protocol/packets/$(DEPDIR)/Interface.Plo \
+	controller/protocol/packets/$(DEPDIR)/Extra.Plo \
 	os/darwin/$(DEPDIR)/AudioInputAudioUnit.Plo \
 	os/darwin/$(DEPDIR)/AudioInputAudioUnitOSX.Plo \
 	os/darwin/$(DEPDIR)/AudioOutputAudioUnit.Plo \
@@ -2401,9 +2401,9 @@ SRC = TgVoip.cpp VoIPController.cpp tools/Buffers.cpp \
 	controller/net/PacketReassembler.cpp \
 	controller/protocol/PacketManager.cpp \
 	controller/protocol/PacketStructs.cpp \
-	controller/protocol/packets/Interface.cpp \
+	controller/protocol/packets/Extra.cpp VoIPGroupController.cpp \
-	VoIPGroupController.cpp VoIPServerConfig.cpp audio/AudioIO.cpp \
+	VoIPServerConfig.cpp audio/AudioIO.cpp audio/AudioInput.cpp \
-	audio/AudioInput.cpp audio/AudioOutput.cpp audio/Resampler.cpp \
+	audio/AudioOutput.cpp audio/Resampler.cpp \
 	audio/AudioInputTester.cpp os/posix/NetworkSocketPosix.cpp \
 	video/VideoSource.cpp video/VideoRenderer.cpp \
 	video/VideoPacketSender.cpp video/VideoFEC.cpp \
@@ -2588,7 +2588,7 @@ controller/protocol/packets/$(am__dirstamp):
 controller/protocol/packets/$(DEPDIR)/$(am__dirstamp):
 	@$(MKDIR_P) controller/protocol/packets/$(DEPDIR)
 	@: > controller/protocol/packets/$(DEPDIR)/$(am__dirstamp)
-controller/protocol/packets/Interface.lo:  \
+controller/protocol/packets/Extra.lo:  \
 	controller/protocol/packets/$(am__dirstamp) \
 	controller/protocol/packets/$(DEPDIR)/$(am__dirstamp)
 audio/$(am__dirstamp):
@@ -4062,7 +4062,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@controller/net/$(DEPDIR)/PacketReassembler.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@controller/protocol/$(DEPDIR)/PacketManager.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@controller/protocol/$(DEPDIR)/PacketStructs.Plo@am__quote@ # am--include-marker
-@AMDEP_TRUE@@am__include@ @am__quote@controller/protocol/packets/$(DEPDIR)/Interface.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@controller/protocol/packets/$(DEPDIR)/Extra.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@os/darwin/$(DEPDIR)/AudioInputAudioUnit.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@os/darwin/$(DEPDIR)/AudioInputAudioUnitOSX.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@os/darwin/$(DEPDIR)/AudioOutputAudioUnit.Plo@am__quote@ # am--include-marker
@@ -4957,7 +4957,7 @@ distclean: distclean-am
 	-rm -f controller/net/$(DEPDIR)/PacketReassembler.Plo
 	-rm -f controller/protocol/$(DEPDIR)/PacketManager.Plo
 	-rm -f controller/protocol/$(DEPDIR)/PacketStructs.Plo
-	-rm -f controller/protocol/packets/$(DEPDIR)/Interface.Plo
+	-rm -f controller/protocol/packets/$(DEPDIR)/Extra.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioInputAudioUnit.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioInputAudioUnitOSX.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioOutputAudioUnit.Plo
@@ -5329,7 +5329,7 @@ maintainer-clean: maintainer-clean-am
 	-rm -f controller/net/$(DEPDIR)/PacketReassembler.Plo
 	-rm -f controller/protocol/$(DEPDIR)/PacketManager.Plo
 	-rm -f controller/protocol/$(DEPDIR)/PacketStructs.Plo
-	-rm -f controller/protocol/packets/$(DEPDIR)/Interface.Plo
+	-rm -f controller/protocol/packets/$(DEPDIR)/Extra.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioInputAudioUnit.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioInputAudioUnitOSX.Plo
 	-rm -f os/darwin/$(DEPDIR)/AudioOutputAudioUnit.Plo

VoIPController.h

@@ -40,6 +40,7 @@
 #include "controller/net/CongestionControl.h"
 #include "controller/protocol/PacketManager.h"
 #include "controller/protocol/PacketStructs.h"
+#include "controller/protocol/protocol/Extra.h"
 #include "tools/Buffers.h"
 #include "controller/net/PacketReassembler.h"
 #include "tools/MessageThread.h"
@@ -398,7 +399,7 @@ public:
     {
         int32_t userID;
         uint8_t id;
-        StreamType type;
+        StreamInfo::StreamType type;
         uint32_t codec;
         bool enabled;
         bool extraECEnabled;
@@ -442,7 +443,7 @@ protected:
     void InitializeTimers();
     void ResetEndpointPingStats();
     void ProcessIncomingVideoFrame(Buffer frame, uint32_t pts, bool keyframe, uint16_t rotation);
-    std::shared_ptr<Stream> GetStreamByType(StreamType type, bool outgoing);
+    std::shared_ptr<Stream> GetStreamByType(StreamInfo::StreamType type, bool outgoing);
     std::shared_ptr<Stream> GetStreamByID(unsigned char id, bool outgoing);
     Endpoint *GetEndpointForPacket(const PendingOutgoingPacket &pkt);
     Endpoint *GetEndpointById(const int64_t id);

controller/PrivateDefines.h

@@ -64,48 +64,15 @@ enum ProtocolVersions
 #define XPFLAG_HAS_RECV_TS 2 // Video calls
 
 
-// Extra signaling packet types
-#define EXTRA_TYPE_STREAM_FLAGS 1
-#define EXTRA_TYPE_STREAM_CSD 2
-#define EXTRA_TYPE_LAN_ENDPOINT 3
-#define EXTRA_TYPE_NETWORK_CHANGED 4
-#define EXTRA_TYPE_GROUP_CALL_KEY 5
-#define EXTRA_TYPE_REQUEST_GROUP 6
-#define EXTRA_TYPE_IPV6_ENDPOINT 7
-
-#define EXTRA_TYPE_INIT 8
-#define EXTRA_TYPE_INIT_ACK 9
-
-// Stream signaling flags
-#define STREAM_FLAG_ENABLED 1
-#define STREAM_FLAG_DTX 2
-#define STREAM_FLAG_EXTRA_EC 4
-#define STREAM_FLAG_PAUSED 8
-
-
 // For codec identifiers
 #define FOURCC(a, b, c, d) ((uint32_t)d | ((uint32_t)c << 8) | ((uint32_t)b << 16) | ((uint32_t)a << 24))
 #define PRINT_FOURCC(x) (char)(x >> 24), (char)(x >> 16), (char)(x >> 8), (char)x
 
 #define CODEC_OPUS_OLD 1
 
-#define CODEC_OPUS FOURCC('O', 'P', 'U', 'S')
-
-#define CODEC_AVC FOURCC('A', 'V', 'C', ' ')
-#define CODEC_HEVC FOURCC('H', 'E', 'V', 'C')
-#define CODEC_VP8 FOURCC('V', 'P', '8', '0')
-#define CODEC_VP9 FOURCC('V', 'P', '9', '0')
-#define CODEC_AV1 FOURCC('A', 'V', '0', '1')
-
 // MTU
 #define DEFAULT_MTU 1100
 
-// Init flags
-#define INIT_FLAG_DATA_SAVING_ENABLED 1
-#define INIT_FLAG_GROUP_CALLS_SUPPORTED 2
-#define INIT_FLAG_VIDEO_SEND_SUPPORTED 4
-#define INIT_FLAG_VIDEO_RECV_SUPPORTED 8
-
 // Video flags
 #define INIT_VIDEO_RES_NONE 0
 #define INIT_VIDEO_RES_240 1

controller/protocol/PacketStructs.cpp

@@ -7,8 +7,47 @@ bool Packet::parse(const BufferInputStream &in, const VersionInfo &ver)
 {
     if (!ver.isNew())
     {
+        legacy = true;
         return parseLegacy(in, ver);
     }
+
+    bool res = in.TryRead(seq) &&
+               in.TryRead(ackSeq) &&
+               in.TryRead(ackMask) &&
+               in.TryRead(flags);
+    if (!res)
+        return false;
+
+    streamId = flags & 3;
+    flags >>= 2;
+
+    uint16_t length;
+    if (!(flags & Flags::Len16 ? in.TryRead(length) : in.TryReadCompat<uint8_t>(length)))
+        return false;
+
+    eFlags = length >> 11;
+    length &= 0x7FF;
+
+    if (eFlags & EFlags::Fragmented)
+    {
+        if (!in.TryRead(fragmentCount))
+            return false;
+        if (!in.TryRead(fragmentIndex))
+            return false;
+    }
+
+    data = Buffer(length);
+    if (in.TryRead(data))
+        return false;
+
+    if ((flags & Flags::RecvTS) && !in.TryRead(recvTS))
+        return false;
+    if ((flags & Flags::ExtraFEC) && !in.TryRead(extraEC, ver))
+        return false;
+    if ((flags & Flags::ExtraSignaling) && !in.TryRead(extraSignaling, ver))
+        return false;
+    
+    return true;
 }
 
 bool Packet::parseLegacy(const BufferInputStream &in, const VersionInfo &ver)
@@ -34,7 +73,7 @@ bool Packet::parseLegacy(const BufferInputStream &in, const VersionInfo &ver)
     {
         return false;
     }
-    this->seq = pseq;
+    this->legacySeq = pseq;
     this->ackSeq = ackId;
     this->ackMask = acks;
 
@@ -60,9 +99,93 @@ bool Packet::parseLegacy(const BufferInputStream &in, const VersionInfo &ver)
     }
     else if (type == PKT_STREAM_EC)
     {
+        if (ver.peerVersion < 7)
+        {
+            uint8_t count;
+            if (!(in.TryRead(streamId) && in.TryRead(seq) && in.TryRead(count)))
+                return false;
+
+            seq /= 60; // Constant frame duration
+
+            for (uint8_t i = 0; i < std::min(count, uint8_t{8}); i++)
+            {
+                if (!in.TryRead(extraEC.v[i], ver))
+                    return false;
+            }
+            for (uint8_t i = count > 8 ? 8 - count : 0; i > 0; i--)
+            {
+                Wrapped<Bytes> ignored;
+                if (!in.TryRead(ignored, ver))
+                    return false;
+            }
+        }
+        else
+        {
+            // Ignore video FEC for now
+        }
     }
     else if (type == PKT_STREAM_DATA || type == PKT_STREAM_DATA_X2 || type == PKT_STREAM_DATA_X3)
     {
+        for (uint8_t count = PKT_STREAM_DATA ? 1 : PKT_STREAM_DATA_X2 ? 2 : 3; count > 0; --count)
+        {
+            Packet *packet = this;
+            if (count > 1)
+            {
+                otherPackets.push_back(Packet());
+                packet = &otherPackets.back();
+                packet->legacy = true;
+            }
+
+            uint8_t flags;
+            uint16_t len;
+            if (!(in.TryRead(packet->streamId) &&
+                          in.TryRead(flags) &&
+                          flags & STREAM_DATA_FLAG_LEN16
+                      ? in.TryRead(len)
+                      : in.TryReadCompat<uint8_t>(len) &&
+                            in.TryRead(packet->seq))) // damn you autoindentation
+                return false;
+
+            packet->seq /= 60; // Constant frame duration
+
+            bool fragmented = static_cast<bool>(len & STREAM_DATA_XFLAG_FRAGMENTED);
+            bool extraFEC = static_cast<bool>(len & STREAM_DATA_XFLAG_EXTRA_FEC);
+            bool keyframe = static_cast<bool>(len & STREAM_DATA_XFLAG_KEYFRAME);
+            if (fragmented)
+            {
+                packet->eFlags |= EFlags::Fragmented;
+                if (!in.TryRead(packet->fragmentIndex))
+                    return false;
+                if (!in.TryRead(packet->fragmentCount))
+                    return false;
+            }
+            if (keyframe)
+            {
+                packet->eFlags |= EFlags::Keyframe;
+            }
+
+            packet->data = Buffer(len & 0x7FF);
+            if (!in.TryRead(packet->data))
+                return false;
+
+            if (extraFEC)
+            {
+                uint8_t count;
+                if (!in.TryRead(count))
+                    return false;
+                for (uint8_t i = 0; i < std::min(count, uint8_t{8}); i++)
+                {
+                    if (!in.TryRead(packet->extraEC.v[i], ver))
+                        return false;
+                }
+                for (uint8_t i = count > 8 ? 8 - count : 0; i > 0; i--)
+                {
+                    Wrapped<Bytes> ignored;
+                    if (!in.TryRead(ignored, ver))
+                        return false;
+                }
+            }
+        }
     }
 }
 bool Packet::parseLegacyLegacy(const BufferInputStream &in, unsigned char &type, uint32_t &ackId, uint32_t &pseq, uint32_t &acks, unsigned char &pflags, size_t &packetInnerLen, int peerVersion)

controller/protocol/PacketStructs.h

@@ -26,6 +26,7 @@ public:
         RecvTS = 2,
         ExtraFEC = 4,
         ExtraSignaling = 8
+
     };
     enum EFlags : uint8_t
     {
@@ -33,20 +34,29 @@ public:
         Keyframe = 2
     };
 
-    uint32_t seq;
+    bool legacy = false;
-    uint32_t ackSeq;
+    uint32_t legacySeq = 0;
-    uint32_t ackMask;
 
-    uint8_t streamId;
+    uint32_t seq = 0;
+    uint32_t ackSeq = 0;
+    uint32_t ackMask = 0;
+
+    uint8_t streamId = 0;
     uint8_t flags = 0;
     uint8_t eFlags = 0;
 
+    uint8_t fragmentIndex = 0;
+    uint8_t fragmentCount = 1;
+
     uint32_t recvTS = 0;
 
     Buffer data;
 
-    Array<Bytes> extraEC;
+    Mask<Wrapped<Bytes>> extraEC;
     Array<Wrapped<Extra>> extraSignaling;
+
+    // Ugly backwards compatibility hacks
+    std::vector<Packet> otherPackets;
 };
 
 // Legacy stuff

controller/protocol/protocol/Extra.h

@@ -8,7 +8,7 @@ namespace tgvoip
 {
 struct Extra : public Serializable, MultiChoice<Extra>
 {
-    static std::shared_ptr<T> chooseFromType(uint8_t type);
+    static std::shared_ptr<Extra> chooseFromType(uint8_t type);
 };
 
 struct Codec : public Serializable, SingleChoice<Codec>
@@ -36,12 +36,12 @@ public:
 
     enum StreamType : uint8_t
     {
-        STREAM_TYPE_AUDIO = 1,
+        Audio = 1,
-        STREAM_TYPE_VIDEO
+        Video
     };
 
     uint8_t streamId = 0;
-    StreamType type = STREAM_TYPE_AUDIO;
+    StreamType type = StreamType::Audio;
     Codec codec;
     uint16_t frameDuration = 0;
     bool enabled = false;

controller/protocol/protocol/Interface.h

@@ -26,7 +26,6 @@ struct SingleChoice
     }
 };
 
-
 template <typename T>
 struct MultiChoice
 {
@@ -40,6 +39,44 @@ public:
     virtual void serialize(BufferOutputStream &out, const VersionInfo &ver) const = 0;
 };
 
+template <typename T>
+struct Mask : public Serializable, SingleChoice<Mask<T>>
+{
+    bool parse(const BufferInputStream &in, const VersionInfo &ver) override
+    {
+        uint8_t mask;
+        if (!in.TryRead(mask))
+        {
+            return false;
+        }
+        for (auto i = 0; i < sizeof(mask); i++)
+        {
+            if (!(i & (1 << i)))
+                continue;
+
+            if (!in.TryRead(v.at(i), ver))
+                return false;
+        }
+        return true;
+    }
+    void serialize(BufferOutputStream &out, const VersionInfo &ver) const override
+    {
+        uint8_t mask = 0;
+        for (auto i = 0; i < sizeof(mask); i++)
+        {
+            if (v[i]) {
+                mask |= 1 << i;
+            }
+        }
+        out.WriteByte(mask);
+        for (const auto &data : v) {
+            out.Write(v, data);
+        }
+    }
+
+    std::array<T, 8> v{};
+};
+
 template <typename T>
 struct Array : public Serializable, SingleChoice<Array<T>>
 {
@@ -77,9 +114,10 @@ struct Array : public Serializable, SingleChoice<Array<T>>
 template <class T>
 struct Wrapped : public Serializable, SingleChoice<Wrapped<T>>
 {
-    Wrapped(std::shared_ptr &&_d) : d(_d);
+    Wrapped(std::shared_ptr<T> &&_d) : d(_d) {};
-    Wrapped(std::shared_ptr &_d) : d(_d);
+    Wrapped(std::shared_ptr<T> &_d) : d(_d) {};
-    
+    Wrapped() = default;
+
     bool parse(const BufferInputStream &in, const VersionInfo &ver) override
     {
         uint8_t len;
@@ -113,9 +151,13 @@ struct Bytes : public Serializable,
     {
         out.WriteBytes(data);
     }
+    operator bool()
+    {
+        return !data.IsEmpty();
+    }
     Buffer data;
 };
-struct UInt32 : public Serializable, SingleChoice<Bytes>
+struct UInt32 : public Serializable, SingleChoice<UInt32>
 {
     bool parse(const BufferInputStream &in, const VersionInfo &ver)
     {

schemeNew.tlb

@@ -16,8 +16,9 @@ packet$_
     length:(## flags.0 ? 11 : (streamId > 0 ? 8 : 0))
     eFlags:(## flags.0 ? 5 : 0) // We shouldn't typically need this, but since we have the space why not use it
     
-    fragmented:eFlags.0?true // Logically, this flag can only appear on *big* fragmented messages
+    fragmentIndex:eFlags.0?(## 8) // Logically, this flag can only appear on *big* fragmented messages
-    keyframe:eFlags.1?true   // This flag can only appear for video streams (typically *big* messages, see ^)
+    fragmentCount:eFlags.0?(## 8) // Logically, this flag can only appear on *big* fragmented messages
+    keyframe:eFlags.1?true        // This flag can only appear for video streams (typically *big* messages, see ^)
 
     data:bytes // No PTS, simply use the seq