-
Notifications
You must be signed in to change notification settings - Fork 181
Expand file tree
/
Copy pathtls_challenge_test.go
More file actions
259 lines (232 loc) · 9.01 KB
/
Copy pathtls_challenge_test.go
File metadata and controls
259 lines (232 loc) · 9.01 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
//go:build test_e2e || test_compatibility
package tests
import (
"context"
_ "embed"
"fmt"
"os"
"regexp"
"testing"
"time"
"github.com/stackrox/rox/generated/storage"
"github.com/stackrox/rox/pkg/namespaces"
"github.com/stackrox/rox/tests/logmatchers"
"github.com/stretchr/testify/suite"
appsV1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
s = namespaces.StackRox // for brevity
proxyNs = "qa-tls-challenge" // Must match the additionalCA X509v3 Subject Alternative Name
proxyImagePullSecretName = "quay"
centralEndpointVar = "ROX_CENTRAL_ENDPOINT"
)
//go:embed "bad-ca/root.crt"
var additionalCA []byte
//go:embed "bad-ca/nginx-loadbalancer.qa-tls-challenge.crt"
var leafCert []byte
//go:embed "bad-ca/nginx-loadbalancer.qa-tls-challenge.key"
var leafKey []byte
func TestTLSChallenge(t *testing.T) {
suite.Run(t, new(TLSChallengeSuite))
}
type TLSChallengeSuite struct {
KubernetesSuite
ctx context.Context
cleanupCtx context.Context
cancel func()
originalCentralEndpoint string
}
func (ts *TLSChallengeSuite) SetupSuite() {
ts.KubernetesSuite.SetupSuite()
ts.ctx, ts.cleanupCtx, ts.cancel = testContexts(ts.T(), "TestTLSChallenge", 15*time.Minute)
// Check sanity before test.
waitUntilCentralSensorConnectionIs(ts.T(), ts.ctx, storage.ClusterHealthStatus_HEALTHY)
ts.logf("Gathering original central endpoint value from sensor...")
ts.originalCentralEndpoint = ts.mustGetDeploymentEnvVal(ts.ctx, s, sensorDeployment, sensorContainer, centralEndpointVar)
ts.logf("Original value is %q. (Will restore this value on cleanup.)", ts.originalCentralEndpoint)
ts.setupProxy(ts.originalCentralEndpoint)
}
func (ts *TLSChallengeSuite) TearDownSuite() {
ts.cleanupProxy(ts.cleanupCtx, proxyNs)
if ts.originalCentralEndpoint != "" {
_ = ts.mustSetDeploymentEnvVal(ts.cleanupCtx, s, sensorDeployment, sensorContainer, centralEndpointVar, ts.originalCentralEndpoint)
}
// Check sanity after test.
waitUntilCentralSensorConnectionIs(ts.T(), ts.cleanupCtx, storage.ClusterHealthStatus_HEALTHY)
ts.cancel()
}
func (ts *TLSChallengeSuite) TestTLSChallenge() {
// This test relies on several log lines appearing in the Sensor logs. One of those does not appear
// when running in 3.74. This is caused by Collector being set to NO_COLLECTION method
// which results in no Collector container in the pod (only Compliance is present).
// That condition makes it impossible for Sensor to parse the Collector image version from
// the Collector container because such container is absent. This sends Sensor into an error state
// and the said log line is never produced. A fix for that is trivial, but would require a patch release for 3.74,
// and we do not do patch releases for this version anymore.
// See getCollectorInfo() in sensor/kubernetes/clusterhealth/updater.go for implementation details
if os.Getenv("COLLECTION_METHOD") == "NO_COLLECTION" {
ts.T().Skipf("The \"COLLECTION_METHOD\" is set to \"NO_COLLECTION\". " +
"For compatibility tests against Sensor version 3.74.x, \"NO_COLLECTION\" is the only valid setting.")
}
const (
proxyServiceName = "nginx-loadbalancer"
proxyEndpoint = proxyServiceName + "." + proxyNs + ":443"
)
ts.logf("Pointing sensor at the proxy...")
patchedDeploy := ts.mustSetDeploymentEnvVal(ts.ctx, s, sensorDeployment, sensorContainer, centralEndpointVar, proxyEndpoint)
ts.waitUntilK8sDeploymentGenerationReady(ts.ctx, s, sensorDeployment, patchedDeploy.GetGeneration())
ts.logf("Sensor will now attempt connecting via the nginx proxy.")
ts.waitUntilLog(ts.ctx, s, map[string]string{"app": "sensor"}, sensorContainer, "contain info about successful connection",
logmatchers.ContainsLineMatching(regexp.MustCompile("Info: Add central CA cert with CommonName: 'Custom Root'")),
logmatchers.ContainsLineMatching(regexp.MustCompile("Info: Connecting to Central server "+proxyEndpoint)),
logmatchers.ContainsLineMatching(regexp.MustCompile("Info: Established connection to Central.")),
logmatchers.ContainsLineMatching(regexp.MustCompile("Info: Communication with central started.")),
)
waitUntilCentralSensorConnectionIs(ts.T(), ts.ctx, storage.ClusterHealthStatus_HEALTHY)
}
func (ts *TLSChallengeSuite) setupProxy(centralEndpoint string) {
name := "nginx-loadbalancer"
nginxLabels := map[string]string{"app": "nginx"}
nginxTLSSecretName := "nginx-tls-conf" //nolint:gosec // G101
nginxConfigName := "nginx-proxy-conf"
ts.logf("Setting up nginx proxy in namespace %q...", proxyNs)
ts.createProxyNamespace()
ts.installImagePullSecret()
ts.createProxyTLSSecret(nginxTLSSecretName)
ts.createProxyConfigMap(centralEndpoint, nginxConfigName)
ts.createService(ts.ctx, proxyNs, name, nginxLabels, map[int32]int32{443: 8443})
ts.createProxyDeployment(name, nginxLabels, nginxConfigName, nginxTLSSecretName)
ts.waitUntilK8sDeploymentReady(ts.ctx, proxyNs, name)
ts.logf("Nginx proxy is now set up in namespace %q.", proxyNs)
}
func (ts *TLSChallengeSuite) createProxyNamespace() {
_, err := ts.k8s.CoreV1().Namespaces().Create(ts.ctx, &v1.Namespace{ObjectMeta: metaV1.ObjectMeta{Name: proxyNs}}, metaV1.CreateOptions{})
if apiErrors.IsAlreadyExists(err) {
return
}
ts.Require().NoError(err, "cannot create proxy namespace %q", proxyNs)
}
func (ts *TLSChallengeSuite) installImagePullSecret() {
ts.ensureQuayImagePullSecretExists(ts.ctx, proxyNs, proxyImagePullSecretName)
}
func (ts *TLSChallengeSuite) createProxyTLSSecret(nginxTLSSecretName string) {
var certChain []byte
certChain = append(certChain, leafCert...)
certChain = append(certChain, additionalCA...)
ts.ensureSecretExists(ts.ctx, proxyNs, nginxTLSSecretName, v1.SecretTypeTLS, map[string][]byte{
v1.TLSCertKey: certChain,
v1.TLSPrivateKeyKey: leafKey,
})
}
func (ts *TLSChallengeSuite) createProxyConfigMap(centralEndpoint string, nginxConfigName string) {
const nginxConfigTmpl = `
server {
listen 8443 ssl http2;
ssl_certificate /run/secrets/tls/tls.crt;
ssl_certificate_key /run/secrets/tls/tls.key;
proxy_temp_path /tmp/nginx_proxy_temp;
client_body_temp_path /tmp/nginx_client_temp;
fastcgi_temp_path /tmp/nginx_fastcgi;
uwsgi_temp_path /tmp/nginx_uwsgi;
scgi_temp_path /tmp/nginx_scgi;
location / {
client_max_body_size 50M;
grpc_pass grpcs://%s;
grpc_ssl_verify off;
}
}
`
ts.ensureConfigMapExists(ts.ctx, proxyNs, nginxConfigName, map[string]string{
"nginx-proxy-grpc-tls.conf": fmt.Sprintf(nginxConfigTmpl, centralEndpoint),
})
}
func (ts *TLSChallengeSuite) createProxyDeployment(name string, nginxLabels map[string]string, nginxConfigName string, nginxTLSSecretName string) {
d := &appsV1.Deployment{
ObjectMeta: metaV1.ObjectMeta{
Name: name,
Labels: nginxLabels,
},
Spec: appsV1.DeploymentSpec{
Selector: &metaV1.LabelSelector{
MatchLabels: nginxLabels,
},
MinReadySeconds: 15,
Template: v1.PodTemplateSpec{
ObjectMeta: metaV1.ObjectMeta{
Labels: nginxLabels,
},
Spec: v1.PodSpec{
ImagePullSecrets: []v1.LocalObjectReference{
{Name: proxyImagePullSecretName},
},
Containers: []v1.Container{
{
Image: "quay.io/rhacs-eng/qa-multi-arch:nginx-1-17-1",
Name: "nginx-loadbalancer",
VolumeMounts: []v1.VolumeMount{
{
Name: "config",
ReadOnly: true,
MountPath: "/etc/nginx/conf.d/",
},
{
Name: "tls",
ReadOnly: true,
MountPath: "/run/secrets/tls",
},
{
Name: "varrun",
MountPath: "/var/run",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "config",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: nginxConfigName,
},
},
},
},
{
Name: "tls",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: nginxTLSSecretName,
},
},
},
{
Name: "varrun",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
},
},
},
},
}
_, err := ts.k8s.AppsV1().Deployments(proxyNs).Create(ts.ctx, d, metaV1.CreateOptions{})
ts.Require().NoError(err, "cannot create deployment %q in namespace %q", name, proxyNs)
}
func (ts *TLSChallengeSuite) cleanupProxy(ctx context.Context, proxyNs string) {
if ts.T().Failed() {
ts.logf("Test failed. Collecting k8s artifacts before cleanup.")
collectLogs(ts.T(), namespaces.StackRox, "tls-challenge-failure")
collectLogs(ts.T(), proxyNs, "tls-challenge-failure")
}
ts.logf("Cleaning up nginx proxy in namespace %q...", proxyNs)
err := ts.k8s.CoreV1().Namespaces().Delete(ctx, proxyNs, metaV1.DeleteOptions{})
if apiErrors.IsNotFound(err) {
return
}
ts.Require().NoError(err, "cannot delete proxy namespace %q", proxyNs)
}