feat(run-integration-test)!: Add OTLP Bearer Auth

run-integration-test/README.md

@@ -94,6 +94,7 @@ profiles:
 | Input                  | Required | Description                                                            |
 | ---------------------- | -------- | ---------------------------------------------------------------------- |
 | `replicated-api-token` | Yes      | Replicated API Token, available via `secrets.REPLICATED_API_TOKEN`     |
+| `otlp-bearer-token`    | Yes      | Bearer token for the OTLP ingester (otlp.stackable.build)              |
 | `test-mode`            | Yes      | Either run a `profile` or a `custom` test                              |
 | `test-mode-input`      | Yes      | The name of the profile or runner, based on the `test-mode`            |
 | `test-suite`           | No       | The name of the BeKu test-suite (only used if running a `custom` test) |

run-integration-test/action.yaml

@@ -17,6 +17,8 @@ inputs:
   # Tokens
   replicated-api-token:
     description: Replicated API token
+  otlp-bearer-token:
+    description: Bearer token for the OTLP ingester (otlp.stackable.build)
 
   # Tool versions
   interu-version:
@@ -38,7 +40,7 @@ inputs:
   helm-version:
     description: Version of helm
     # See https://github.com/helm/helm/releases for latest version
-    default: v3.19.0
+    default: v3.21.1
   stackablectl-version:
     description: Version of stackablectl
     # See https://github.com/stackabletech/stackable-cockpit/releases for latest version
@@ -62,7 +64,6 @@ runs:
     - name: Install interu
       env:
         INTERU_VERSION: ${{ inputs.interu-version }}
-        RUNNER_DEBUG: ${{ runner.debug }}
       shell: bash
       run: "$GITHUB_ACTION_PATH/../.scripts/actions/install_interu.sh"
 
@@ -73,16 +74,16 @@ runs:
         TEST_SUITE: ${{ inputs.test-suite }}
         TEST_MODE: ${{ inputs.test-mode }}
         TEST: ${{ inputs.test }}
-        RUNNER_DEBUG: ${{ runner.debug }}
         GITHUB_REPOSITORY: ${{ github.repository }}
+        GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}"
         GITHUB_RUN_ID: ${{ github.run_id }}"
       shell: bash
       run: |
         set -euo pipefail
         [ -n "${RUNNER_DEBUG+set}" ] && set -x
 
         # Generate the cluster name
-        echo "KUBERNETES_CLUSTER_NAME=integration-test-${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}" | tee -a "$GITHUB_OUTPUT"
+        echo "KUBERNETES_CLUSTER_NAME=integration-test-${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | tee -a "$GITHUB_OUTPUT"
 
         # Run interu to expand parameters into GITHUB_OUTPUT for use as env vars in later steps.
         if [ "$TEST_MODE" == "profile" ]; then
@@ -105,7 +106,8 @@ runs:
         KUBECTL_VERSION: ${{ inputs.kubectl-version }}
         KUTTL_VERSION: ${{ inputs.kuttl-version }}
         HELM_VERSION: ${{ inputs.helm-version }}
-        RUNNER_DEBUG: ${{ runner.debug }}
+        # The helm signature is expired since a couple of years...
+        VERIFY_SIGNATURE: "false"
       shell: bash
       run: |
         "$GITHUB_ACTION_PATH/../.scripts/actions/install_kubectl.sh"
@@ -145,7 +147,7 @@ runs:
 
     - name: Prepare Replicated Cluster
       id: prepare-replicated-cluster
-      uses: replicatedhq/replicated-actions/create-cluster@291bef61a059631e39e84f8470f86152171c4c20 # v1.26.0
+      uses: replicatedhq/replicated-actions/create-cluster@0fbf5395cf3e8c744f447bbc92b7de9135eead8e # v1.27.1
       with:
         # See: https://github.com/replicatedhq/replicated-actions/tree/main/create-cluster#inputs
         api-token: ${{ inputs.replicated-api-token }}
@@ -196,6 +198,24 @@ runs:
         kubectl -n opentelemetry-operator get pods
         echo "::endgroup::"
 
+    - name: Install OTLP ingester auth Secret
+      shell: bash
+      env:
+        OTLP_BEARER_TOKEN: ${{ inputs.otlp-bearer-token }}
+      run: |
+        set -euo pipefail
+        # No 'set -x' here: it would risk exposing the token. Mask defensively in case the
+        # value did not arrive via secrets.* (which GitHub redacts automatically).
+        echo "::add-mask::${OTLP_BEARER_TOKEN}"
+
+        echo "::group::kubectl apply secret"
+        # The collectors read this via spec.env -> secretKeyRef (otlp-auth / bearer-token).
+        kubectl create secret generic otlp-auth \
+          --namespace opentelemetry-operator \
+          --from-literal=bearer-token="${OTLP_BEARER_TOKEN}" \
+          --dry-run=client -o yaml | kubectl apply -f -
+        echo "::endgroup::"
+
     - name: Apply OpenTelemetry Collectors configurations
       shell: bash
       env:
@@ -205,6 +225,7 @@ runs:
         INTERU_KUBERNETES_DISTRIBUTION: ${{ steps.extract.outputs.INTERU_KUBERNETES_DISTRIBUTION }}
         INTERU_KUBERNETES_VERSION: ${{ steps.extract.outputs.INTERU_KUBERNETES_VERSION }}
         GITHUB_TRIGGERED_BY: ${{ github.triggering_actor }}
+        GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
       run: |
         set -euo pipefail
 
@@ -214,6 +235,7 @@ runs:
         echo "KUBERNETES_DISTRIBUTION=${INTERU_KUBERNETES_DISTRIBUTION}" | tee -a "$INTEGRATION_TEST_INFO"
         echo "KUBERNETES_VERSION=${INTERU_KUBERNETES_VERSION}" | tee -a "$INTEGRATION_TEST_INFO"
         echo "TRIGGERED_BY=${GITHUB_TRIGGERED_BY}" | tee -a "$INTEGRATION_TEST_INFO"
+        echo "RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT}" | tee -a "$INTEGRATION_TEST_INFO"
         echo "::endgroup::"
 
         echo "::group::kubectl apply"
@@ -240,6 +262,9 @@ runs:
         REF_NAME: ${{ github.ref_name }}
         GH_TOKEN: ${{ github.token }}
         OPERATOR_NAME: ${{ steps.extract_operator_name.outputs.OPERATOR_NAME }}
+        BEKU_TEST_PARALLELISM: ${{ steps.extract.outputs.BEKU_TEST_PARALLELISM }}
+        BEKU_TEST_SUITE: ${{ steps.extract.outputs.BEKU_TEST_SUITE }}
+        BEKU_TEST: ${{ steps.extract.outputs.BEKU_TEST }}
       shell: bash
       run: |
         set -euo pipefail
@@ -263,7 +288,7 @@ runs:
       if: always()
       # If the creation of the cluster failed, we don't want to error and abort
       continue-on-error: true
-      uses: replicatedhq/replicated-actions/remove-cluster@291bef61a059631e39e84f8470f86152171c4c20 # v1.26.0
+      uses: replicatedhq/replicated-actions/remove-cluster@0fbf5395cf3e8c744f447bbc92b7de9135eead8e # v1.27.1
       with:
         # See: https://github.com/replicatedhq/replicated-actions/tree/main/remove-cluster#inputs
         api-token: ${{ inputs.replicated-api-token }}

run-integration-test/kustomize/.gitignore

@@ -1 +1,4 @@
 charts/
+
+# Local-only secrets (eg: overlays/local/otlp-auth.secret.env). Never commit tokens.
+*.secret.env

run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml

@@ -36,7 +36,7 @@ spec:
   config:
     receivers:
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/filelogreceiver
-      filelog/varlogpods:
+      file_log/varlogpods:
         # A storage extension (eg: redis, or file) can be used for storing log offsets. Otherwise it is held in memory.
         # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/storage
         # storage: {}
@@ -105,7 +105,7 @@ spec:
 
     processors:
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor
-      k8sattributes:
+      k8s_attributes:
         filter:
           # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor/README.md#as-an-agent
           node_from_env_var: KUBE_NODE_NAME # this should be same as the var set from the downward API further up
@@ -124,7 +124,7 @@ spec:
         # This key exists so it can be easily extended in an overlay:
         attributes: []
 
-      resourcedetection/env:
+      resource_detection/env:
         detectors: [env]
         timeout: 2s
         override: false
@@ -146,8 +146,8 @@ spec:
       extensions: []
       pipelines:
         logs:
-          receivers: [filelog/varlogpods]
+          receivers: [file_log/varlogpods]
           # processors: [memory_limiter, batch]
-          processors: [k8sattributes, resourcedetection/env, resource]
+          processors: [k8s_attributes, resource_detection/env, resource]
           # Enable configured exporters in the overlay
           exporters: []

run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml

@@ -29,7 +29,7 @@ spec:
 
     processors:
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor
-      k8sattributes:
+      k8s_attributes:
         # Use the k8s attributes set by the receiver
         # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor#as-a-gateway
         passthrough: true
@@ -51,7 +51,7 @@ spec:
             key: service.name
             value: kubernetes-events
 
-      resourcedetection/env:
+      resource_detection/env:
         detectors: [env]
         timeout: 2s
         override: false
@@ -75,12 +75,12 @@ spec:
         logs:
           receivers: [k8s_events]
           # processors: [memory_limiter, batch]
-          processors: [k8sattributes, resourcedetection/env, resource]
+          processors: [k8s_attributes, resource_detection/env, resource]
           # Enable configured exporters in the overlay
           exporters: []
         metrics:
           receivers: [k8s_cluster]
           # processors: [memory_limiter, batch]
-          processors: [k8sattributes, resourcedetection/env, resource]
+          processors: [k8s_attributes, resource_detection/env, resource]
           # Enable configured exporters in the overlay
           exporters: []

run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml

@@ -27,7 +27,7 @@ spec:
 
     processors:
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor
-      k8sattributes:
+      k8s_attributes:
         # Use the k8s attributes set by the receiver
         # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor#as-a-gateway
         passthrough: false
@@ -49,7 +49,7 @@ spec:
           key: service.name
           value: kubernetes-objects
 
-      resourcedetection/env:
+      resource_detection/env:
         detectors: [env]
         timeout: 2s
         override: false
@@ -73,6 +73,6 @@ spec:
         logs:
           receivers: [k8sobjects]
           # processors: [memory_limiter, batch]
-          processors: [k8sattributes, resourcedetection/env, resource]
+          processors: [k8s_attributes, resource_detection/env, resource]
           # Enable configured exporters in the overlay
           exporters: []

run-integration-test/kustomize/bases/opentelemetry-operator/kustomization.yml

@@ -9,7 +9,10 @@ helmCharts:
 - name: opentelemetry-operator
   repo: https://open-telemetry.github.io/opentelemetry-helm-charts
   # Find the latest release here: https://github.com/open-telemetry/opentelemetry-helm-charts/releases
-  version: 0.93.0
+  # helm repo add opentelemetry-helm-charts https://open-telemetry.github.io/opentelemetry-helm-charts --force-update
+  # helm search repo opentelemetry-helm-charts/opentelemetry-operator
+  # renovate: registryUrl=https://open-telemetry.github.io/opentelemetry-helm-charts
+  version: 0.120.0 # 0.156.0
   releaseName: opentelemetry-operator
   includeCRDs: true
   skipTests: true
@@ -22,7 +25,7 @@ helmCharts:
         # See which plugins the opentelemetry-collector-k8s image contains here:
         # https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-k8s/manifest.yaml
         repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s
-        tag: 0.132.4 # Often the chart has old values for the collector image
+        tag: 0.157.0 # Often the chart has old values for the collector image
     admissionWebhooks:
       certManager:
         enabled: false

run-integration-test/kustomize/overlays/local/kustomization.yml

@@ -0,0 +1,102 @@
+---
+# Local testing overlay: a self-contained sibling of ../replicated for running on a
+# local cluster (kind/minikube/k3d) without the run-integration-test GitHub Action.
+#
+# Differences from ../replicated:
+# - Resource attributes are set to static literal values instead of ${env:...} vars
+#   populated by the CI action, so no integration-test-info.env / configMapGenerator
+#   is needed (the collector errors on startup if an ${env:...} var is unset).
+# - Adds a `debug` exporter to the logs pipeline so telemetry is visible on the
+#   collector pod's stdout locally (kubectl logs) in addition to the OTLP/HTTP export.
+namePrefix: local-
+
+resources:
+- ../../bases/opentelemetry-collectors
+
+patches:
+# Update all collectors to set the exporter endpoints and any other common overrides.
+- target:
+    group: opentelemetry.io
+    version: v1beta1
+    kind: OpenTelemetryCollector
+  patch: |-
+    - op: add
+      path: /spec/config/exporters/otlp_http
+      value:
+        # otlp_http appends the signal-specific path (eg: /v1/logs) to this endpoint.
+        endpoint: https://otlp.stackable.build
+        auth:
+          authenticator: bearertokenauth
+    - op: add
+      path: /spec/config/exporters/debug
+      value:
+        verbosity: detailed
+    # The ingester at otlp.stackable.build requires Bearer auth. The token is read from
+    # the OTLP_BEARER_TOKEN env var, sourced from the otlp-auth Secret (spec/env below).
+    # Locally that Secret is built by the secretGenerator from a gitignored otlp-auth.env.
+    - op: add
+      path: /spec/config/extensions/bearertokenauth
+      value:
+        token: ${env:OTLP_BEARER_TOKEN}
+    - op: add
+      path: /spec/config/service/extensions/-
+      value: bearertokenauth
+    - op: add
+      path: /spec/env/-
+      value:
+        name: OTLP_BEARER_TOKEN
+        valueFrom:
+          secretKeyRef:
+            name: otlp-auth
+            key: bearer-token
+    - op: add
+      path: /spec/config/service/pipelines/logs/exporters/-
+      value: otlp_http
+    - op: add
+      path: /spec/config/service/pipelines/logs/exporters/-
+      value: debug
+    - op: add
+      path: /spec/config/processors/resource/attributes/-
+      value:
+        action: upsert
+        key: k8s.cluster.name
+        value: local
+    - op: add
+      path: /spec/config/processors/resource/attributes/-
+      value:
+        action: upsert
+        key: k8s.cluster.distribution
+        value: local
+    - op: add
+      path: /spec/config/processors/resource/attributes/-
+      value:
+        action: upsert
+        key: k8s.cluster.version
+        value: local
+    - op: add
+      path: /spec/config/processors/resource/attributes/-
+      value:
+        action: upsert
+        key: github.actions.triggered_by
+        value: local
+
+# Specifically override config for the kubernetes-events collector.
+- target:
+    group: opentelemetry.io
+    version: v1beta1
+    kind: OpenTelemetryCollector
+    name: kubernetes-events
+  patch: |-
+    # OpenSearch Metrics collector is not yet available
+    - op: remove
+      path: /spec/config/service/pipelines/metrics
+
+secretGenerator:
+# Local-only: builds the otlp-auth Secret from a gitignored otlp-auth.secret.env so the
+# token is never committed. Copy otlp-auth.secret.env.example to otlp-auth.secret.env and
+# set the real token. The generated (hashed) Secret name is rewritten into each collector's
+# spec.env via the Secret nameReference in ../../bases/opentelemetry-collectors/crds.yml.
+- name: otlp-auth
+  namespace: opentelemetry-operator
+  envs:
+  - otlp-auth.secret.env

run-integration-test/kustomize/overlays/local/otlp-auth.secret.env.example

@@ -0,0 +1,4 @@
+# Copy this file to otlp-auth.secret.env and set the real Bearer token for
+# otlp.stackable.build. Files matching *.secret.env are gitignored and must never
+# be committed.
+bearer-token=REPLACE_WITH_BEARER_TOKEN