TLS 1.2 Negotiation on Windows with ECDSA server certificate (#6803)

Resolves #6467 

It appears .NET clients on Windows 10 cannot use TLS 1.3, and when
negotiating down to TLS 1.2 the signature format is more strict,
requiring ASN.1 DER encoding, producing a larger signature than the one
generated using the more permissive TLS 1.3 signature.

This change increases the signature buffer to exactly half of the hash
size.

Signed-off-by: John Weldon <jweldon@synadia.com>

server/certstore/certstore_windows.go

@@ -593,7 +593,7 @@ func winSignECDSA(kh uintptr, digest []byte) ([]byte, error) {
 		return nil, ErrStoreECDSASigningError
 	}
 
-	return winPackECDSASigValue(bytes.NewReader(buf[:size]), len(digest))
+	return winPackECDSASigValue(bytes.NewReader(buf[:size]), int(size/2))
 }
 
 func winPackECDSASigValue(r io.Reader, digestLength int) ([]byte, error) {

server/certstore_windows_test.go

@@ -16,6 +16,7 @@
 package server
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/url"
 	"os"
@@ -186,7 +187,7 @@ func TestLeafTLSWindowsCertStore(t *testing.T) {
 }
 
 // TestServerTLSWindowsCertStore tests the topology of a NATS server requiring TLS and gettings it own server
-// cert identiy (as used when accepting NATS client connections and negotiating TLS) from Windows certificate store.
+// cert identity (as used when accepting NATS client connections and negotiating TLS) from Windows certificate store.
 func TestServerTLSWindowsCertStore(t *testing.T) {
 
 	// Server Identity (server.pem)
@@ -307,3 +308,43 @@ func TestServerIgnoreExpiredCerts(t *testing.T) {
 		})
 	}
 }
+
+func TestWindowsTLS12ECDSA(t *testing.T) {
+	err := runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", []string{"ecdsa_server.pfx"})
+	if err != nil {
+		t.Fatalf("expected powershell provision to succeed: %v", err)
+	}
+
+	config := createConfFile(t, []byte(`
+	listen: "localhost:-1"
+	tls {
+		cert_store: "WindowsCurrentUser"
+		cert_match_by: "Thumbprint"
+		cert_match: "4F8AF21756E5DBBD54619BBB6F3CC5D455ED4468"
+		cert_match_skip_invalid: true
+		timeout: 5
+	}
+	`))
+	defer removeFile(t, config)
+
+	srv, _ := RunServerWithConfig(config)
+	if srv == nil {
+		t.Fatalf("expected to be able start server with cert store configuration")
+	}
+	defer srv.Shutdown()
+
+	for name, version := range map[string]uint16{
+		"TLS 1.3": tls.VersionTLS13,
+		"TLS 1.2": tls.VersionTLS12,
+	} {
+		t.Run(name, func(t *testing.T) {
+			tc := &tls.Config{MaxVersion: version, MinVersion: version, InsecureSkipVerify: true}
+
+			if _, err = nats.Connect(srv.clientConnectURLs[0], nats.Secure(tc)); err != nil {
+				t.Fatalf("connection with %s: %v", name, err)
+			}
+
+			t.Logf("successful connection with %s", name)
+		})
+	}
+}

test/configs/certs/tlsauth/certstore/ecdsa_server.key

@@ -0,0 +1,6 @@
+-----BEGIN PRIVATE KEY-----
+MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAfrq5ri+W7sYQp/6xc
+lH6YbTy43dnnrKUbbdlzsDn4DPzO1k15LVXx8EPK+7vuh5uhZANiAAR6V4nqBt3k
+ZfO9H664fPB8PkuDhphBfzxbSFFcr2DXj11g0ZV56Yjnh3RMC4Lud29ofpTQd8IP
+9bspEvjnBvOw60tH9WiquWqxLgSREUZVLEMD1dZ3JSVUfDCI2zzf00s=
+-----END PRIVATE KEY-----

test/configs/certs/tlsauth/certstore/ecdsa_server.pem

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB2jCCAWCgAwIBAgIUKRYMoky98mN3mpyL6PMIY8/d2OswCgYIKoZIzj0EAwMw
+FjEUMBIGA1UEAwwLbmF0cy1zZXJ2ZXIwHhcNMjUwMjA3MTgxNDM5WhcNMzUwMjA1
+MTgxNDM5WjAWMRQwEgYDVQQDDAtuYXRzLXNlcnZlcjB2MBAGByqGSM49AgEGBSuB
+BAAiA2IABHpXieoG3eRl870frrh88Hw+S4OGmEF/PFtIUVyvYNePXWDRlXnpiOeH
+dEwLgu53b2h+lNB3wg/1uykS+OcG87DrS0f1aKq5arEuBJERRlUsQwPV1nclJVR8
+MIjbPN/TS6NvMG0wHQYDVR0OBBYEFHbcfCfGs+l2bVg22WLTdV10AnpTMB8GA1Ud
+IwQYMBaAFHbcfCfGs+l2bVg22WLTdV10AnpTMA8GA1UdEwEB/wQFMAMBAf8wGgYD
+VR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49BAMDA2gAMGUCMQDhzRyw
+Q+m2fMFyqIgFc890jLIzh2bGqlmdkUpb+/Z/y9zKZQPSG5xhXp7A/FhvM24CMHVW
+ZIWBCJJRhw/L3s73QHX1d+M6mNqES16cnnht6j9DF1AddIipcsnBcpo4s7K/Xg==
+-----END CERTIFICATE-----

test/configs/certs/tlsauth/certstore/generate_ecdsa_test_cert.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+SCRIPT_ROOT="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+cert_file_prefix="${SCRIPT_ROOT}/ecdsa_server"
+export_password="s3cr3t"
+
+openssl req -x509 \
+  -days 3650 \
+  -newkey ec \
+  -pkeyopt ec_paramgen_curve:secp384r1 \
+  -sha384 \
+  -subj "/CN=nats-server" \
+  --addext "subjectAltName=IP:127.0.0.1,DNS:localhost" \
+  -nodes \
+  -out "${cert_file_prefix}.pem" \
+  -keyout "${cert_file_prefix}.key" \
+  -outform PEM >/dev/null 2>&1
+
+openssl pkcs12 \
+  -inkey "${cert_file_prefix}.key" \
+  -in "${cert_file_prefix}.pem" \
+  -export \
+  -password "pass:${export_password}" \
+  -out "${cert_file_prefix}.pfx" >/dev/null 2>&1