FIPS 140-3 mode support

When enabling FIPS 140-3 mode with `GODEBUG=gofips=X` on supported platforms,
the following restrictions are made to allow NATS to function:

1. `auth_callout` cannot be configured and will error at startup if it is;
2. `chacha` filestore encryption mode cannot be configured and will
   error at startup if it is;
3. `X25519` is removed from the default curve preferences;
4. TLS handshakes that require non-FIPS-compliant algorithms will fail.

Signed-off-by: Neil Twigg <neil@nats.io>

server/ciphersuites.go

@@ -14,6 +14,7 @@
 package server
 
 import (
+	"crypto/fips140"
 	"crypto/tls"
 )
 
@@ -52,6 +53,16 @@ var curvePreferenceMap = map[string]tls.CurveID{
 // reorder to default to the highest level of security.  See:
 // https://blog.bracebin.com/achieving-perfect-ssl-labs-score-with-go
 func defaultCurvePreferences() []tls.CurveID {
+	if fips140.Enabled() {
+		// X25519 is not FIPS-approved by itself, but it is when
+		// combined with MLKEM768.
+		return []tls.CurveID{
+			tls.X25519MLKEM768, // post-quantum
+			tls.CurveP256,
+			tls.CurveP384,
+			tls.CurveP521,
+		}
+	}
 	return []tls.CurveID{
 		tls.X25519MLKEM768, // post-quantum
 		tls.X25519,         // faster than P256, arguably more secure

server/opts.go

@@ -15,6 +15,7 @@ package server
 
 import (
 	"context"
+	"crypto/fips140"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
@@ -2484,6 +2485,9 @@ func parseJetStreamTPM(v interface{}, opts *Options, errors *[]error) error {
 func setJetStreamEkCipher(opts *Options, mv interface{}, tk token) error {
 	switch strings.ToLower(mv.(string)) {
 	case "chacha", "chachapoly":
+		if fips140.Enabled() {
+			return &configErr{tk, fmt.Sprintf("Cipher type %q cannot be used in FIPS-140 mode", mv)}
+		}
 		opts.JetStreamCipher = ChaCha
 	case "aes":
 		opts.JetStreamCipher = AES
@@ -4378,6 +4382,10 @@ func parseAuthorization(v any, errors, warnings *[]error) (*authorization, error
 			}
 			auth.defaultPermissions = permissions
 		case "auth_callout", "auth_hook":
+			if fips140.Enabled() {
+				*errors = append(*errors, fmt.Errorf("'auth_callout' cannot be configured in FIPS-140 mode"))
+				continue
+			}
 			ac, err := parseAuthCallout(tk, errors)
 			if err != nil {
 				*errors = append(*errors, err)

server/server.go

@@ -16,6 +16,7 @@ package server
 import (
 	"bytes"
 	"context"
+	"crypto/fips140"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
@@ -722,8 +723,12 @@ func NewServer(opts *Options) (*Server, error) {
 	pub, _ := kp.PublicKey()
 
 	// Create an xkey for encrypting messages from this server.
-	xkp, _ := nkeys.CreateCurveKeys()
-	xpub, _ := xkp.PublicKey()
+	var xkp nkeys.KeyPair
+	var xpub string
+	if !fips140.Enabled() {
+		xkp, _ = nkeys.CreateCurveKeys()
+		xpub, _ = xkp.PublicKey()
+	}
 
 	serverName := pub
 	if opts.ServerName != _EMPTY_ {