test: add basic job for fips system-tests (#16174)

* test: add basic job for fips system-tests

* test: simplify systemtest apm-server build func

* test: update certificates for fips compliance

* test: generate subject key id for fips compliance

* test: update TLS protocol error message

* Update tls_test.go

* Update command.go

* Update command.go

* test: remove unnecessary files and update tls tests

* Update tls_nofips_test.go

* Update systemtest/apmservertest/server.go

Co-authored-by: Eric <22215921+ericywl@users.noreply.github.com>

---------

Co-authored-by: Eric <22215921+ericywl@users.noreply.github.com>

Author: kruskall

.github/workflows/ci.yml

@@ -96,6 +96,25 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: make system-test
 
+  system-test-fips:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: systemtest/go.mod
+          cache: true
+          cache-dependency-path: |
+            go.sum
+            systemtest/go.sum
+      - run: docker compose up -d
+      - env:
+          GOTESTFLAGS: "-v -tags=requirefips"
+          GOFIPS140: "latest"
+          GODEBUG: "fips140=only"
+          GH_TOKEN: ${{ github.token }}
+        run: make system-test
+
   test-package:
     runs-on: ubuntu-latest
     steps:

systemtest/apmservertest/cert.pem

@@ -19,6 +19,7 @@ package apmservertest
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"log"
 	"os"
@@ -168,15 +169,22 @@ func BuildServerBinary(goos, goarch string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	relpath := filepath.Join("build", fmt.Sprintf("apm-server-%s-%s", goos, goarch))
+	name := "apm-server"
+	abspath := filepath.Join(repoRoot, "build", fmt.Sprintf("apm-server-%s-%s", goos, goarch))
+	if fips140.Enabled() {
+		abspath += "-fips"
+		name += "-fips"
+	}
 	if goos == "windows" {
-		relpath += ".exe"
+		abspath += ".exe"
 	}
-	abspath := filepath.Join(repoRoot, relpath)
 
-	log.Println("Building apm-server...")
-	cmd := exec.Command("make", relpath)
+	log.Printf("Building %s...", name)
+	cmd := exec.Command("make", name)
 	cmd.Dir = repoRoot
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "NOCP=1") // prevent race condition
+	cmd.Env = append(cmd.Env, "GOOS="+goos, "GOARCH="+goarch)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {

systemtest/apmservertest/client_cert.pem

@@ -20,18 +20,24 @@ package apmservertest
 import (
 	"bytes"
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -60,6 +66,11 @@ type Server struct {
 	// The temporary directory will be removed when the server is closed.
 	Dir string
 
+	// CertDir is the directory of the TLS certificates.
+	//
+	// If Dir is empty the current directory is used
+	CertDir string
+
 	// Log holds an optional io.Writer to which the process's Stderr will
 	// be written, in addition to being available through the Server.Logs
 	// field.
@@ -125,6 +136,7 @@ func NewServerTB(tb testing.TB, args ...string) *Server {
 // will be written under apm-server/systemtest/logs/<test-name>/.
 func NewUnstartedServerTB(tb testing.TB, args ...string) *Server {
 	s := NewUnstartedServer(args...)
+	s.CertDir = tb.TempDir()
 	logfile := createLogfile(tb, "apm-server")
 	s.Log = logfile
 	tb.Cleanup(func() {
@@ -253,14 +265,8 @@ func (s *Server) start(tls bool) error {
 }
 
 func (s *Server) initTLS() (serverCertPath, serverKeyPath, caCertPath string, _ error) {
-	repoRoot, err := getRepoRoot()
-	if err != nil {
-		panic(err)
-	}
-
+	serverCertPath, serverKeyPath, err := generateCerts(s.CertDir, true, x509.ExtKeyUsageServerAuth, "127.0.0.1", "::1")
 	// Load a self-signed server certificate for testing TLS encryption.
-	serverCertPath = filepath.Join(repoRoot, "systemtest", "apmservertest", "cert.pem")
-	serverKeyPath = filepath.Join(repoRoot, "systemtest", "apmservertest", "key.pem")
 	serverCertBytes, err := os.ReadFile(serverCertPath)
 	if err != nil {
 		return "", "", "", err
@@ -270,9 +276,12 @@ func (s *Server) initTLS() (serverCertPath, serverKeyPath, caCertPath string, _
 		panic("failed to add CA certificate to cert pool")
 	}
 
+	clientCertPath, clientKeyPath, err := generateCerts(s.CertDir, false, x509.ExtKeyUsageClientAuth, "")
+	if err != nil {
+		return "", "", "", err
+	}
+
 	// Load a self-signed client certificate for testing TLS client certificate auth.
-	clientCertPath := filepath.Join(repoRoot, "systemtest", "apmservertest", "client_cert.pem")
-	clientKeyPath := filepath.Join(repoRoot, "systemtest", "apmservertest", "client_key.pem")
 	clientCertBytes, err := os.ReadFile(clientCertPath)
 	if err != nil {
 		return "", "", "", err
@@ -289,11 +298,83 @@ func (s *Server) initTLS() (serverCertPath, serverKeyPath, caCertPath string, _
 	s.TLS = &tls.Config{
 		Certificates: []tls.Certificate{clientCert},
 		RootCAs:      certpool,
-		MinVersion:   tls.VersionTLS12,
 	}
 	return serverCertPath, serverKeyPath, clientCertPath, nil
 }
 
+func generateCerts(dir string, ca bool, keyUsage x509.ExtKeyUsage, hosts ...string) (string, string, error) {
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return "", "", fmt.Errorf("Failed to generate serial number: %w", err)
+	}
+	notBefore := time.Now()
+	notAfter := notBefore.Add(24 * time.Hour)
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"Org"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{keyUsage},
+		BasicConstraintsValid: true,
+	}
+
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		}
+	}
+
+	if ca {
+		template.IsCA = true
+		template.KeyUsage |= x509.KeyUsageCertSign
+	}
+
+	clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to generate client key: %w", err)
+	}
+	privBytes, err := x509.MarshalPKCS8PrivateKey(clientKey)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to marshal private key: %w", err)
+	}
+
+	h := sha256.Sum256(privBytes)
+	template.SubjectKeyId = h[:]
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, clientKey.Public(), clientKey)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to create certificate: %w", err)
+	}
+	certOut, err := os.CreateTemp(dir, "client_cert.pem")
+	if err != nil {
+		return "", "", fmt.Errorf("failed to open client_cert.pem for writing: %w", err)
+	}
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return "", "", fmt.Errorf("failed to write data to client_cert.pem: %w", err)
+	}
+	if err := certOut.Close(); err != nil {
+		return "", "", fmt.Errorf("error closing client_cert.pem: %w", err)
+	}
+
+	keyOut, err := os.CreateTemp(dir, "client_key.pem")
+	if err != nil {
+		return "", "", fmt.Errorf("failed to open client_key.pem for writing: %w", err)
+	}
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		return "", "", fmt.Errorf("failed to write data to client_key.pem: %w", err)
+	}
+	if err := keyOut.Close(); err != nil {
+		return "", "", fmt.Errorf("error closing client_key.pem: %w", err)
+	}
+
+	return certOut.Name(), keyOut.Name(), nil
+}
+
 func (s *Server) printCmdline(w io.Writer, args []string) {
 	var buf bytes.Buffer
 	fmt.Fprint(&buf, "# Running apm-server\n")

systemtest/apmservertest/client_key.pem

@@ -0,0 +1,62 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !requirefips
+
+package systemtest_test
+
+import (
+	"crypto/tls"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-server/systemtest/apmservertest"
+)
+
+func TestDefaultIncompatibleTLSConfig(t *testing.T) {
+	srv := apmservertest.NewUnstartedServerTB(t)
+	require.NoError(t, srv.StartTLS())
+
+	attemptRequest := func(t *testing.T, minVersion, maxVersion uint16, cipherSuites ...uint16) error {
+		tlsConfig := &tls.Config{RootCAs: srv.TLS.RootCAs}
+		tlsConfig.MinVersion = minVersion
+		tlsConfig.MaxVersion = maxVersion
+		tlsConfig.CipherSuites = cipherSuites
+		httpClient := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}
+		resp, err := httpClient.Get(srv.URL)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		return nil
+	}
+
+	t.Run("incompatible_cipher_suite", func(t *testing.T) {
+		err := attemptRequest(t, tls.VersionTLS12, tls.VersionTLS12, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA)
+		require.Error(t, err)
+		assert.Regexp(t, ".*tls: handshake failure", err.Error())
+	})
+
+	t.Run("incompatible_protocol", func(t *testing.T) {
+		err := attemptRequest(t, tls.VersionTLS10, tls.VersionTLS10)
+		require.Error(t, err)
+		assert.Regexp(t, ".*tls: protocol version not supported", err.Error())
+	})
+}