Skip to content

Commit 8b46fb7

Browse files
kb-newrelickb-newrelic
authored
test: migrate slow tests to e2e action (#353)
1 parent d92f348 commit 8b46fb7

38 files changed

+329
-945
lines changed

‎.github/renovate.json5‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,16 @@
99
{
1010
"customType": "regex",
1111
"fileMatch": [
12-
"(^|/)test-spec.yaml$"
12+
"(^|/)spec-nightly.yaml$",
13+
"(^|/)spec-local.yaml$",
1314
],
1415
"matchStringsStrategy": "combination",
1516
"matchStrings": [
16-
".*# renovatebot.*\\s*name:\\s+newrelic\/(?<depName>nr-k8s-otel-collector)\\s+version:\\s+(?:&[\\w_]+\\s+)?(?<currentValue>[\\d\\.]+)",
17+
".*# renovatebot.*\\s+(?:name:\\s+)?newrelic\/(?<depName>nr-k8s-otel-collector)(?:.*\\s+version)?:\\s+(?<currentValue>[\\d\\.]+)",
1718
],
1819
"registryUrlTemplate": "https://helm-charts.newrelic.com",
1920
"datasourceTemplate": "helm",
2021
"versioningTemplate": "semver"
21-
}
22+
},
2223
]
2324
}

‎.github/workflows/ci-base.yaml‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ on:
3131
required: false
3232
nr_account_id:
3333
required: false
34-
nr_api_base_url:
35-
required: false
3634
nr_api_key:
3735
required: false
3836

@@ -157,20 +155,6 @@ jobs:
157155
license_key: ${{ secrets.nr_ingest_key }}
158156
region: 'US'
159157

160-
- name: Run slow local tests
161-
env:
162-
KIND_CLUSTER_NAME: ${{ inputs.test_cluster_name }}
163-
IMAGE_TAG: ${{ env.image_tag }}
164-
IMAGE_REPO: ${{ secrets.registry }}/${{ inputs.distribution}}
165-
DISTRO: ${{ inputs.distribution }}
166-
NR_BACKEND_URL: ${{ secrets.nr_backend_url }}
167-
NR_INGEST_KEY: ${{ secrets.nr_ingest_key }}
168-
NR_API_KEY: ${{ secrets.nr_api_key }}
169-
NR_ACCOUNT_ID: ${{ secrets.nr_account_id }}
170-
NR_API_BASE_URL: ${{ secrets.nr_api_base_url }}
171-
run: |
172-
make -f ./test/e2e/Makefile ci_test-slow
173-
174158
- name: Trivy security check
175159
if: ${{ !inputs.nightly }}
176160
uses: aquasecurity/[email protected]

‎.github/workflows/ci.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ jobs:
3232
nr_backend_url: ${{ secrets.NR_STAGING_BACKEND_URL }}
3333
nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }}
3434
nr_account_id: ${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }}
35-
nr_api_base_url: ${{ secrets.NR_STAGING_API_BASE_URL }}
3635
nr_api_key: ${{ secrets.OTELCOMM_NR_API_KEY }}
3736

3837
terraform:

‎.github/workflows/lint-renovate-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
# info log shows what files and dep ach 'manager' matched
2525
INFO_LOG=$(LOG_LEVEL=info renovate --platform=local)
2626
# we currently expect exactly 1 file and 1 dep
27-
if ! grep 'regex' <<< "${INFO_LOG}" | grep 'fileCount\": 1, \"depCount\": 1'; then
27+
if ! grep 'regex' <<< "${INFO_LOG}" | grep 'fileCount\": 2, \"depCount\": 2'; then
2828
echo "renovate didn't match k8s test-spec"
2929
DEBUG_LOG=$(LOG_LEVEL=debug renovate --platform=local)
3030
echo "Debug log:\n${DEBUG_LOG}"

‎Makefile.dev‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ci:
55
ci_custom_matrix:
66
@# repeat --matrix arg for multiple distros
77
act push -W .github/workflows/ci.yaml \
8-
--matrix distribution:nrdot-collector-host
8+
--matrix distribution:nrdot-collector-k8s
99

1010
ci_nightly_custom_matrix:
1111
@# repeat --matrix arg for multiple distros

‎distributions/nrdot-collector-host/test/host-collector-values.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
additionalLabels:
2+
e2eTestLogSelector: nrdot-collector-host-e2e
3+
podLabels:
4+
e2eTestLogSelector: nrdot-collector-host-e2e
5+
6+
mode: daemonset
7+
8+
configMap:
9+
create: false
10+
11+
command:
12+
extraArgs:
13+
- --config=/etc/nrdot-collector-host/config.yaml
14+
# configure liveliness probe according to chart's expectations
15+
- '"--config=yaml:extensions::health_check::endpoint: ${env:MY_POD_IP}:13133"'
16+
17+
extraEnvs:
18+
- name: OTEL_EXPORTER_OTLP_ENDPOINT
19+
valueFrom:
20+
secretKeyRef:
21+
name: collector-secrets
22+
key: nrBackendUrl
23+
- name: NEW_RELIC_LICENSE_KEY
24+
valueFrom:
25+
secretKeyRef:
26+
name: collector-secrets
27+
key: nrIngestKey
28+
- name: SCENARIO_TAG
29+
valueFrom:
30+
secretKeyRef:
31+
name: collector-secrets
32+
key: scenarioTag
33+
- name: OTEL_RESOURCE_ATTRIBUTES
34+
value: "testKey=$(SCENARIO_TAG)"

‎distributions/nrdot-collector-host/test/host-expected-metrics.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
entities:
2+
- entityType: EXT-SERVICE
3+
metrics:
4+
- name: system.cpu.load_average.15m
5+
- name: system.cpu.load_average.1m
6+
- name: system.cpu.load_average.5m
7+
- name: system.cpu.utilization
8+
- name: system.disk.io
9+
- name: system.disk.io_time
10+
- name: system.disk.operation_time
11+
- name: system.disk.operations
12+
- name: system.filesystem.inodes.usage
13+
- name: system.filesystem.usage
14+
- name: system.filesystem.utilization
15+
- name: system.memory.usage
16+
- name: system.memory.utilization
17+
- name: system.network.dropped
18+
- name: system.network.errors
19+
- name: system.network.io
20+
- name: system.network.packets
21+
- name: system.paging.operations

‎distributions/nrdot-collector-host/test/collector-values-internal-telemetry.yaml‎ renamed to ‎distributions/nrdot-collector-host/test/internal-collector-values.yaml‎

File renamed without changes.

‎distributions/nrdot-collector-host/test/expected-metrics-internal.yaml‎ renamed to ‎distributions/nrdot-collector-host/test/internal-expected-metrics.yaml‎

File renamed without changes.

‎distributions/nrdot-collector-host/test/spec-local.yaml‎

Lines changed: 163 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,166 @@ description: nrdot-collector-host E2E Test
33
# scoping is achieved via an implicit attribute `testKey=${SCENARIO_TAG}` expected on all tested telemetry
44

55
scenarios:
6-
- description: receiver-based + internal Telemetry
6+
- description: host telemetry
7+
before:
8+
- kubectl create ns nr-${SCENARIO_TAG}
9+
- |
10+
kubectl create secret generic 'collector-secrets' --namespace=nr-${SCENARIO_TAG} \
11+
--from-literal="nrIngestKey=${LICENSE_KEY}" --from-literal="nrBackendUrl=${NR_BACKEND_URL}" \
12+
--from-literal="serviceName=nrdot-collector-host-e2e-host" --from-literal="scenarioTag=${SCENARIO_TAG}"
13+
- "helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts || echo 'skipping repo add: open-telemetry repo already exists'"
14+
- |
15+
chart_version=$(../../../scripts/get-otel-collector-chart-version.sh)
16+
helm upgrade --install ${SCENARIO_TAG} open-telemetry/opentelemetry-collector \
17+
--namespace nr-${SCENARIO_TAG} --version ${chart_version} \
18+
--values ./host-collector-values.yaml \
19+
--set image.tag=${IMAGE_TAG} \
20+
--set image.repository=${IMAGE_REPO}
21+
after:
22+
- kubectl logs -l e2eTestLogSelector=nrdot-collector-host-e2e -n nr-${SCENARIO_TAG} --all-containers --prefix=true
23+
- kubectl get all -o wide
24+
- helm uninstall ${SCENARIO_TAG} --namespace nr-${SCENARIO_TAG}
25+
- kubectl delete namespace nr-${SCENARIO_TAG}
26+
tests:
27+
# Tests for existence of metrics
28+
metrics:
29+
- source: "host-expected-metrics.yaml"
30+
# Tests for existence of signal + specific attributes
31+
nrqls:
32+
- query: FROM Metric SELECT filter(count(*), state='user') as state_user WHERE metricName = 'system.cpu.utilization'
33+
expected_results:
34+
- key: state_user
35+
lowerBoundedValue: 1
36+
- query: FROM Metric SELECT filter(count(*), state='user') as state_user WHERE metricName = 'system.cpu.utilization'
37+
expected_results:
38+
- key: state_user
39+
lowerBoundedValue: 1
40+
- query: FROM Metric SELECT filter(count(*), state='idle') as state_idle WHERE metricName = 'system.cpu.utilization'
41+
expected_results:
42+
- key: state_idle
43+
lowerBoundedValue: 1
44+
- query: FROM Metric SELECT filter(count(*), state='wait') as state_wait WHERE metricName = 'system.cpu.utilization'
45+
expected_results:
46+
- key: state_wait
47+
lowerBoundedValue: 1
48+
- query: FROM Metric SELECT filter(count(*), state='steal') as state_steal WHERE metricName = 'system.cpu.utilization'
49+
expected_results:
50+
- key: state_steal
51+
lowerBoundedValue: 1
52+
- query: FROM Metric SELECT filter(count(*), direction='read') as direction_read WHERE metricName = 'system.disk.io'
53+
expected_results:
54+
- key: direction_read
55+
lowerBoundedValue: 1
56+
- query: FROM Metric SELECT filter(count(*), direction='write') as direction_write WHERE metricName = 'system.disk.io'
57+
expected_results:
58+
- key: direction_write
59+
lowerBoundedValue: 1
60+
- query: FROM Metric SELECT filter(count(*), direction='read') as direction_read WHERE metricName = 'system.disk.operation_time'
61+
expected_results:
62+
- key: direction_read
63+
lowerBoundedValue: 1
64+
- query: FROM Metric SELECT filter(count(*), direction='write') as direction_write WHERE metricName = 'system.disk.operation_time'
65+
expected_results:
66+
- key: direction_write
67+
lowerBoundedValue: 1
68+
- query: FROM Metric SELECT filter(count(*), direction='read') as direction_read WHERE metricName = 'system.disk.operations'
69+
expected_results:
70+
- key: direction_read
71+
lowerBoundedValue: 1
72+
- query: FROM Metric SELECT filter(count(*), direction='write') as direction_write WHERE metricName = 'system.disk.operations'
73+
expected_results:
74+
- key: direction_write
75+
lowerBoundedValue: 1
76+
- query: FROM Metric SELECT filter(count(*), state='cached') as state_cached WHERE metricName = 'system.memory.usage'
77+
expected_results:
78+
- key: state_cached
79+
lowerBoundedValue: 1
80+
- query: FROM Metric SELECT filter(count(*), state='free') as state_free WHERE metricName = 'system.memory.usage'
81+
expected_results:
82+
- key: state_free
83+
lowerBoundedValue: 1
84+
- query: FROM Metric SELECT filter(count(*), state='slab_reclaimable') as state_slab_reclaimable WHERE metricName = 'system.memory.usage'
85+
expected_results:
86+
- key: state_slab_reclaimable
87+
lowerBoundedValue: 1
88+
- query: FROM Metric SELECT filter(count(*), state='buffered') as state_buffered WHERE metricName = 'system.memory.usage'
89+
expected_results:
90+
- key: state_buffered
91+
lowerBoundedValue: 1
92+
- query: FROM Metric SELECT filter(count(*), state='used') as state_used WHERE metricName = 'system.memory.usage'
93+
expected_results:
94+
- key: state_used
95+
lowerBoundedValue: 1
96+
- query: FROM Metric SELECT filter(count(*), state='free') as state_free WHERE metricName = 'system.memory.utilization'
97+
expected_results:
98+
- key: state_free
99+
lowerBoundedValue: 1
100+
- query: FROM Metric SELECT filter(count(*), state='used') as state_used WHERE metricName = 'system.memory.utilization'
101+
expected_results:
102+
- key: state_used
103+
lowerBoundedValue: 1
104+
- query: FROM Metric SELECT filter(count(*), direction='page_out') as direction_page_out WHERE metricName = 'system.paging.operations'
105+
expected_results:
106+
- key: direction_page_out
107+
lowerBoundedValue: 1
108+
- query: FROM Metric SELECT filter(count(*), direction='page_in') as direction_page_in WHERE metricName = 'system.paging.operations'
109+
expected_results:
110+
- key: direction_page_in
111+
lowerBoundedValue: 1
112+
- query: FROM Metric SELECT filter(count(*), state='free') as state_free WHERE metricName = 'system.filesystem.inodes.usage'
113+
expected_results:
114+
- key: state_free
115+
lowerBoundedValue: 1
116+
- query: FROM Metric SELECT filter(count(*), state='used') as state_used WHERE metricName = 'system.filesystem.inodes.usage'
117+
expected_results:
118+
- key: state_used
119+
lowerBoundedValue: 1
120+
- query: FROM Metric SELECT filter(count(*), state='used') as state_used WHERE metricName = 'system.filesystem.usage'
121+
expected_results:
122+
- key: state_used
123+
lowerBoundedValue: 1
124+
- query: FROM Metric SELECT filter(count(*), state='free') as state_free WHERE metricName = 'system.filesystem.usage'
125+
expected_results:
126+
- key: state_free
127+
lowerBoundedValue: 1
128+
- query: FROM Metric SELECT filter(count(*), type != 'squashfs') as type_not_squashfs WHERE metricName = 'system.filesystem.utilization'
129+
expected_results:
130+
- key: type_not_squashfs
131+
lowerBoundedValue: 1
132+
- query: FROM Metric SELECT filter(count(*), direction='receive') as direction_receive WHERE metricName = 'system.network.dropped'
133+
expected_results:
134+
- key: direction_receive
135+
lowerBoundedValue: 1
136+
- query: FROM Metric SELECT filter(count(*), direction='transmit') as direction_transmit WHERE metricName = 'system.network.dropped'
137+
expected_results:
138+
- key: direction_transmit
139+
lowerBoundedValue: 1
140+
- query: FROM Metric SELECT filter(count(*), direction='receive') as direction_receive WHERE metricName = 'system.network.errors'
141+
expected_results:
142+
- key: direction_receive
143+
lowerBoundedValue: 1
144+
- query: FROM Metric SELECT filter(count(*), direction='transmit') as direction_transmit WHERE metricName = 'system.network.errors'
145+
expected_results:
146+
- key: direction_transmit
147+
lowerBoundedValue: 1
148+
- query: FROM Metric SELECT filter(count(*), direction='receive') as direction_receive WHERE metricName = 'system.network.io'
149+
expected_results:
150+
- key: direction_receive
151+
lowerBoundedValue: 1
152+
- query: FROM Metric SELECT filter(count(*), direction='transmit') as direction_transmit WHERE metricName = 'system.network.io'
153+
expected_results:
154+
- key: direction_transmit
155+
lowerBoundedValue: 1
156+
- query: FROM Metric SELECT filter(count(*), direction='receive') as direction_receive WHERE metricName = 'system.network.packets'
157+
expected_results:
158+
- key: direction_receive
159+
lowerBoundedValue: 1
160+
- query: FROM Metric SELECT filter(count(*), direction='transmit') as direction_transmit WHERE metricName = 'system.network.packets'
161+
expected_results:
162+
- key: direction_transmit
163+
lowerBoundedValue: 1
164+
165+
- description: receiver-based + internal telemetry
7166
before:
8167
- kubectl create ns nr-${SCENARIO_TAG}
9168
- cp ../../../examples/internal-telemetry-config.yaml ./relay
@@ -14,17 +173,10 @@ scenarios:
14173
--from-literal="serviceName=nrdot-collector-host-e2e-internal" --from-literal="scenarioTag=${SCENARIO_TAG}"
15174
- "helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts || echo 'skipping repo add: open-telemetry repo already exists'"
16175
- |
17-
collector_component_version=$(cat ../manifest.yaml | grep otlpreceiver | awk '{print $NF}')
18-
chart_version=$(helm search repo open-telemetry/opentelemetry-collector --output=json --versions | jq -r ".[] | select (.app_version==\"${collector_component_version:1}\") | .version" | head -n1 )
19-
if [[ -z "${chart_version}" ]]; then
20-
chart_version="$(helm search repo open-telemetry/opentelemetry-collector --output=json | jq '.[0].version')"
21-
echo "Chart matching collector component version ${collector_component_version} wasn't released (yet?). Using latest (${chart_version}) instead."
22-
else
23-
echo "Using chart version: ${chart_version}"
24-
fi
176+
chart_version=$(../../../scripts/get-otel-collector-chart-version.sh)
25177
helm upgrade --install ${SCENARIO_TAG} open-telemetry/opentelemetry-collector \
26178
--namespace nr-${SCENARIO_TAG} --version ${chart_version} \
27-
--values ./collector-values-internal-telemetry.yaml \
179+
--values ./internal-collector-values.yaml \
28180
--set image.tag=${IMAGE_TAG} \
29181
--set image.repository=${IMAGE_REPO}
30182
after:
@@ -35,7 +187,7 @@ scenarios:
35187
tests:
36188
# Tests for existence of metrics
37189
metrics:
38-
- source: "expected-metrics-internal.yaml"
190+
- source: "internal-expected-metrics.yaml"
39191
# Tests for existence of signal + specific attributes
40192
nrqls:
41193
# Logs
@@ -127,7 +279,3 @@ scenarios:
127279
- key: rpc_code
128280
lowerBoundedValue: 1
129281

130-
131-
132-
133-

0 commit comments

Comments
 (0)