net-libs/libvncserver: fix openssl crypto backend

Bug: https://bugs.gentoo.org/893608
Signed-off-by: Alexander Tsoy <alexander@tsoy.me>
Closes: https://github.com/gentoo/gentoo/pull/30311
Signed-off-by: Sam James <sam@gentoo.org>

2 files changed, 140 insertions, 0 deletions

diff --git a/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch b/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch
new file mode 100644
index 000000000000..d4c76773b02c
--- /dev/null
+++ b/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch
@@ -0,0 +1,59 @@
+From b686f379c34114cf938fe88291f58014337558f6 Mon Sep 17 00:00:00 2001
+From: Gaurav Ujjwal <gujjwal00@gmail.com>
+Date: Mon, 23 Jan 2023 00:03:03 +0530
+Subject: [PATCH] common/crypto_openssl: pad DH key buffers with leading zeros
+ for smaller keys
+
+Re: https://github.com/LibVNC/libvncserver/issues/493
+Re: https://github.com/bk138/multivnc/issues/202
+---
+ common/crypto_openssl.c | 18 ++++++++++++++++--
+ 1 file changed, 16 insertions(+), 2 deletions(-)
+
+diff --git a/common/crypto_openssl.c b/common/crypto_openssl.c
+index 50e8073a..dc1ee093 100644
+--- a/common/crypto_openssl.c
++++ b/common/crypto_openssl.c
+@@ -156,6 +156,15 @@ int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], cons
+     return result;
+ }
+ 
++static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {
++    if (current_len >= expected_len || expected_len < 1)
++        return;
++
++    size_t diff = expected_len - current_len;
++    memmove(out + diff, out, current_len);
++    memset(out, 0, diff);
++}
++
+ int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
+ {
+     int result = 0;
+@@ -184,6 +193,9 @@ int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen,
+ 	goto out;
+     if(BN_bn2bin(dh->pub_key, pub_out) == 0)
+ 	goto out;
++
++    pad_leading_zeros(priv_out, BN_num_bytes(dh->priv_key), keylen);
++    pad_leading_zeros(pub_out, BN_num_bytes(dh->pub_key), keylen);
+ #else
+     DH_get0_key(dh, &pub_key, &priv_key);
+     if(BN_bn2binpad(priv_key, priv_out, keylen) == -1)
+@@ -216,9 +228,11 @@ int dh_compute_shared_key(uint8_t *shared_out, const uint8_t *priv, const uint8_
+     if(!DH_set0_key(dh, NULL, BN_bin2bn(priv, keylen, NULL)))
+ 	goto out;
+ #endif
+-    if(DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh) == -1)
+-	goto out;
++    int shared_len = DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh);
++    if(shared_len == -1)
++        goto out;
+ 
++    pad_leading_zeros(shared_out, shared_len, keylen);
+     result = 1;
+ 
+  out:
+-- 
+2.39.2
+
diff --git a/net-libs/libvncserver/libvncserver-0.9.14-r1.ebuild b/net-libs/libvncserver/libvncserver-0.9.14-r1.ebuild
new file mode 100644
index 000000000000..996b5e9c2b4e
--- /dev/null
+++ b/net-libs/libvncserver/libvncserver-0.9.14-r1.ebuild
@@ -0,0 +1,81 @@
+# Copyright 1999-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+inherit cmake
+
+MY_P="LibVNCServer-${PV}"
+
+DESCRIPTION="library for creating vnc servers"
+HOMEPAGE="https://libvnc.github.io/"
+SRC_URI="https://github.com/LibVNC/${PN}/archive/${MY_P}.tar.gz"
+S="${WORKDIR}/${PN}-${MY_P}"
+
+# common/d3des.*: https://github.com/LibVNC/libvncserver/issues/88
+LICENSE="GPL-2 GPL-2+ LGPL-2.1+ BSD MIT"
+# no sub slot wanted (yet), see #578958
+SLOT="0"
+KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~loong ~mips ~ppc ~ppc64 ~riscv ~s390 ~sparc ~x86 ~amd64-linux ~x86-linux"
+IUSE="+24bpp +filetransfer +gcrypt gnutls ipv6 +jpeg lzo +png sasl ssl systemd +threads +zlib"
+# https://bugs.gentoo.org/690202
+# https://bugs.gentoo.org/435326
+# https://bugs.gentoo.org/550916
+REQUIRED_USE="
+	filetransfer? ( threads )
+	jpeg? ( zlib )
+	png? ( zlib )
+	ssl? ( !gnutls? ( threads ) )
+"
+# Avoid using internal crypto backend as it doesn't support
+# all authentication methods #893608
+REQUIRED_USE+="
+	ssl? ( gnutls? ( gcrypt ) )
+	!ssl? ( gcrypt )
+"
+
+DEPEND="
+	gcrypt? ( >=dev-libs/libgcrypt-1.5.3:0= )
+	ssl? (
+		!gnutls? (
+			>=dev-libs/openssl-1.0.2:0=
+		)
+		gnutls? ( >=net-libs/gnutls-2.12.23-r6:0= )
+	)
+	jpeg? ( media-libs/libjpeg-turbo:= )
+	lzo? ( dev-libs/lzo )
+	png? ( >=media-libs/libpng-1.6.10:0= )
+	sasl? ( dev-libs/cyrus-sasl )
+	systemd? ( sys-apps/systemd:= )
+	zlib? ( >=sys-libs/zlib-1.2.8-r1:0= )
+"
+RDEPEND="${DEPEND}"
+
+DOCS=( AUTHORS ChangeLog NEWS.md README.md )
+
+PATCHES=(
+	"${FILESDIR}"/${P}-crypto-openssl-fix.patch
+)
+
+src_configure() {
+	local mycmakeargs=(
+		-DWITH_EXAMPLES=OFF
+		-DWITH_FFMPEG=OFF
+		-DWITH_GTK=OFF
+		-DWITH_SDL=OFF
+		-DWITH_24BPP=$(usex 24bpp ON OFF)
+		-DWITH_TIGHTVNC_FILETRANSFER=$(usex filetransfer ON OFF)
+		-DWITH_GCRYPT=$(usex gcrypt ON OFF)
+		-DWITH_GNUTLS=$(usex gnutls $(usex ssl ON OFF) OFF)
+		-DWITH_IPv6=$(usex ipv6 ON OFF)
+		-DWITH_JPEG=$(usex jpeg ON OFF)
+		-DWITH_LZO=$(usex lzo ON OFF)
+		-DWITH_OPENSSL=$(usex gnutls OFF $(usex ssl ON OFF))
+		-DWITH_PNG=$(usex png ON OFF)
+		-DWITH_SASL=$(usex sasl ON OFF)
+		-DWITH_SYSTEMD=$(usex systemd ON OFF)
+		-DWITH_THREADS=$(usex threads ON OFF)
+		-DWITH_ZLIB=$(usex zlib ON OFF)
+	)
+	cmake_src_configure
+}