Configuring tracing with envoy

Networking
1

We’re trying to get tracing going in our cluster without having the application do the forwarding of b3 and requestId headers from an incoming to an outgoing request. What we have so far are two Envoy filters: an inbound filter to cache the tracing headers by a special header we use tracing incoming requests, and an outbound filter that replaces / appends the tracing framework’s headers (Zipkin in this case) in the outgoing requests.

This should allow us to circumvent any changes in application logic. The issue is that the headers set by the outbound filter seem to get replaced by Envoy at the end - is there a way to disable that or get around it? For reference, the inbound and outbound Envoy filters are below where Lua is used to extract/set headers:

inbound_filter.yaml

{{ if .Values.tracing.enabled }}
apiVersion: networking.istio.io/v1alpha3
kind: EnvoyFilter
metadata:
  name: tracing-inbound
  namespace: istio-system
spec:
  workloadSelector:
    labels:
      { app: "myApp" }
  configPatches:
  - applyTo: HTTP_FILTER
    match:
      context: SIDECAR_INBOUND
      listener:
        filterChain:
          filter:
            name: envoy.filters.network.http_connection_manager
            subFilter:
              name: envoy.filters.http.router
    patch:
      operation: INSERT_BEFORE
      value:
        name: envoy.lua
        typed_config:
          '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua
          inlineCode: |-
            function envoy_on_request(handle)
                -- cache all the tracing headers by x-iot-correlation in the sidecar
                local headers, body = handle:httpCall("outbound|6060||network.local", {
                    [":method"] = "GET",
                    [":path"] = "/incoming",
                    [":authority"] = "",
                    ["x-iot-correlation"] = handle:headers():get("x-iot-correlation"),
                    ["x-b3-traceid"] = handle:headers():get("x-b3-traceid"),
                    ["x-request-id"] = handle:headers():get("x-request-id"),
                    ["x-b3-parentspanid"] = handle:headers():get("x-b3-parentspanid")
                }, "", 5000)
            end

            function envoy_on_response(handle)
            end
{{ end }}

outbound_filter.yaml

{{ if .Values.tracing.enabled }}
apiVersion: networking.istio.io/v1alpha3
kind: EnvoyFilter
metadata:
  name: tracing-outbound
  namespace: istio-system
spec:
  workloadSelector:
    labels:
      { app: "myApp" }
  configPatches:
  - applyTo: HTTP_FILTER
    match:
      context: SIDECAR_OUTBOUND
      listener:
        filterChain:
          filter:
            name: envoy.filters.network.http_connection_manager
            subFilter:
              name: envoy.filters.http.router
    patch:
      operation: INSERT_BEFORE
      value:
        name: envoy.lua
        typed_config:
          '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua
          inlineCode: |-
            function envoy_on_request(handle)
                -- fetch the spanId correlated with a cv if one exists from the sidecar
                -- this call returns all the b3 headers and requestid header in the response headers
                local headers, body = handle:httpCall("outbound|6060||network.local", {
                    [":method"] = "GET",
                    [":path"] = "/outgoing",
                    [":authority"] = "",
                    ["x-iot-correlation"] = handle:headers():get("x-iot-correlation")
                }, "", 5000)
                for key, value in pairs(headers) do
                  handle:headers():replace(key, value)
                end
            end

            function envoy_on_response(handle)
            end
{{ end }}