rtmp: fix RTMPE handshake error when a public key starts with zero (#…

…2269)
bluenviron · Aug 30, 2023 · fc353ce · fc353ce
1 parent f69be81
commit fc353ce
Showing 1 changed file with 12 additions and 1 deletion.
diff --git a/internal/rtmp/handshake/dh.go b/internal/rtmp/handshake/dh.go
@@ -1,6 +1,7 @@
 package handshake
 
 import (
+	"bytes"
 	"crypto/rand"
 	"fmt"
 	"math/big"
@@ -100,6 +101,10 @@ func dhGenerateKeyPair() ([]byte, []byte, error) {
 	y.Exp(&g, &x, &p)
 	pub := y.Bytes()
 
+	if len(pub) < dhKeyLength {
+		pub = append(bytes.Repeat([]byte{0}, dhKeyLength-len(pub)), pub...)
+	}
+
 	return priv, pub, nil
 }
 
@@ -114,5 +119,11 @@ func dhComputeSharedSecret(priv []byte, pub []byte) []byte {
 	p.SetBytes(p1024)
 	var z big.Int
 	z.Exp(&y, &x, &p)
-	return z.Bytes()
+	sec := z.Bytes()
+
+	if len(sec) < dhKeyLength {
+		sec = append(bytes.Repeat([]byte{0}, dhKeyLength-len(sec)), sec...)
+	}
+
+	return sec
 }