I’m making a request and the Authorization header is getting removed and as a result it can’t be properly evaluated by the RequestAuthentication config.
Here’s the curl I’m making
curl --location --request GET 'http://localhost:8001/api/v1/namespaces/istio-system/services/istio-ingressgateway:80/proxy/ping' --header 'Authorization: Bearer {token}'
There’s a couple things to notice from the logs which are the core of my issue:
- There’s no Authorization: Bearer {token} header
- Because the auth header is being striped, the values from istio_authn aren’t being listed and thus I can’t write any logic for my AuthorizationPolicy.
If I use fromHeaders and change which header I read the token from this issue gets resolved, but I want to forward the Authorization header to the service and I’m just generally curious why my Authorization header is missing.
Here are the logs I’m getting:
kubectl logs $(kubectl -n istio-system get pods -l istio=ingressgateway -o jsonpath='{.items[0].metadata.name}') -c istio-proxy -n istio-system
[2021-04-03T03:13:09.125Z] "GET /ping HTTP/1.1" 200 - via_upstream - "-" 0 1181 233 228 "127.0.0.1, 172.18.0.1,10.244.0.1" "curl/7.64.1" "bb97a356-2f42-4f45-befe-0a8a2c64b220" "localhost:8001" "10.45.255.76:8080" outbound|8080||www.example.com 10.244.0.8:45756 10.244.0.8:8080 10.244.0.1:43598 - -
2021-04-03T03:14:02.350318Z debug envoy rbac checking request: requestedServerName: , sourceIP: 10.244.0.1:43598, directRemoteIP: 10.244.0.1:43598, remoteIP: 10.244.0.1:43598,localAddress: 10.244.0.8:8080, ssl: none, headers: ':authority', 'localhost:8001'
':path', '/ping'
':method', 'GET'
'user-agent', 'curl/7.64.1'
'accept', '*/*'
'accept-encoding', 'gzip'
'x-forwarded-for', '127.0.0.1, 172.18.0.1,10.244.0.1'
'x-forwarded-uri', '/api/v1/namespaces/istio-system/services/istio-ingressgateway:80/proxy/ping'
'x-forwarded-proto', 'http'
'x-envoy-external-address', '10.244.0.1'
'x-request-id', '7985329d-e449-4319-9ac1-28e88248cb49'
'x-envoy-decorator-operation', 'www.example.com:8080/*'
'x-envoy-peer-metadata', '{removed}'
'x-envoy-peer-metadata-id', 'router~10.244.0.8~istio-ingressgateway-7cc49dcd99-xmhfh.istio-system~istio-system.svc.cluster.local'
, dynamicMetadata: filter_metadata {
key: "istio_authn"
value {
}
}
2021-04-03T03:14:02.350364Z debug envoy rbac enforced allowed, matched policy ns[istio-system]-policy[frontend-ingress]-rule[0]
[2021-04-03T03:14:02.350Z] "GET /ping HTTP/1.1" 200 - via_upstream - "-" 0 1181 37 36 "127.0.0.1, 172.18.0.1,10.244.0.1" "curl/7.64.1" "7985329d-e449-4319-9ac1-28e88248cb49" "localhost:8001" "10.45.255.76:8080" outbound|8080||www.example.com 10.244.0.8:45756 10.244.0.8:8080 10.244.0.1:43598 - -
Proxy config.
[
{
"name": "0.0.0.0_8080",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 8080
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"statPrefix": "outbound_0.0.0.0_8080",
"rds": {
"configSource": {
"ads": {},
"resourceApiVersion": "V3"
},
"routeConfigName": "http.80"
},
"httpFilters": [
{
"name": "istio.metadata_exchange",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm",
"value": {
"config": {
"configuration": {
"@type": "type.googleapis.com/google.protobuf.StringValue",
"value": "{}\n"
},
"vm_config": {
"code": {
"local": {
"inline_string": "envoy.wasm.metadata_exchange"
}
},
"runtime": "envoy.wasm.runtime.null"
}
}
}
}
},
{
"name": "envoy.filters.http.jwt_authn",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication",
"providers": {
"origins-0": {
"issuer": "https://auth.example.com",
"localJwks": {
"inlineString": "{removed}"
},
"payloadInMetadata": "https://auth.example.com"
}
},
"rules": [
{
"match": {
"prefix": "/"
},
"requires": {
"requiresAny": {
"requirements": [
{
"providerName": "origins-0"
},
{
"allowMissing": {}
}
]
}
}
}
]
}
},
{
"name": "istio_authn",
"typedConfig": {
"@type": "type.googleapis.com/istio.envoy.config.filter.http.authn.v2alpha1.FilterConfig",
"policy": {
"origins": [
{
"jwt": {
"issuer": "https://auth.example.com"
}
}
],
"originIsOptional": true,
"principalBinding": "USE_ORIGIN"
},
"skipValidateTrustDomain": true
}
},
{
"name": "envoy.filters.http.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC",
"rules": {
"policies": {
"ns[istio-system]-policy[frontend-ingress]-rule[0]": {
"permissions": [
{
"andRules": {
"rules": [
{
"any": true
}
]
}
}
],
"principals": [
{
"andIds": {
"ids": [
{
"any": true
}
]
}
}
]
}
}
}
}
},
{
"name": "envoy.filters.http.cors",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors"
}
},
{
"name": "envoy.filters.http.fault",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault"
}
},
{
"name": "istio.stats",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm",
"value": {
"config": {
"configuration": {
"@type": "type.googleapis.com/google.protobuf.StringValue",
"value": "{\n \"debug\": \"false\",\n \"stat_prefix\": \"istio\",\n \"disable_host_header_fallback\": true,\n \"metrics\": [\n {\n \"dimensions\": {\n \"source_cluster\": \"node.metadata['CLUSTER_ID']\",\n \"destination_cluster\": \"upstream_peer.cluster_id\"\n }\n }\n ]\n}\n"
},
"root_id": "stats_outbound",
"vm_config": {
"code": {
"local": {
"inline_string": "envoy.wasm.stats"
}
},
"runtime": "envoy.wasm.runtime.null",
"vm_id": "stats_outbound"
}
}
}
}
},
{
"name": "envoy.filters.http.router",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"clientSampling": {
"value": 100
},
"randomSampling": {
"value": 1
},
"overallSampling": {
"value": 100
},
"customTags": [
{
"tag": "istio.canonical_revision",
"environment": {
"name": "CANONICAL_REVISION",
"defaultValue": "latest"
}
},
{
"tag": "istio.canonical_service",
"environment": {
"name": "CANONICAL_SERVICE",
"defaultValue": "unknown"
}
},
{
"tag": "istio.mesh_id",
"environment": {
"name": "ISTIO_META_MESH_ID",
"defaultValue": "unknown"
}
},
{
"tag": "istio.namespace",
"environment": {
"name": "POD_NAMESPACE",
"defaultValue": "default"
}
}
]
},
"httpProtocolOptions": {},
"serverName": "istio-envoy",
"streamIdleTimeout": "0s",
"accessLog": [
{
"name": "envoy.access_loggers.file",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog",
"path": "/dev/stdout",
"logFormat": {
"textFormat": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS% %CONNECTION_TERMINATION_DETAILS% \"%UPSTREAM_TRANSPORT_FAILURE_REASON%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME% %ROUTE_NAME%\n"
}
}
}
],
"useRemoteAddress": true,
"forwardClientCertDetails": "SANITIZE_SET",
"setCurrentClientCertDetails": {
"subject": true,
"cert": true,
"dns": true,
"uri": true
},
"upgradeConfigs": [
{
"upgradeType": "websocket"
}
],
"normalizePath": true
}
}
]
}
],
"trafficDirection": "OUTBOUND",
"accessLog": [
{
"name": "envoy.access_loggers.file",
"filter": {
"responseFlagFilter": {
"flags": [
"NR"
]
}
},
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog",
"path": "/dev/stdout",
"logFormat": {
"textFormat": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS% %CONNECTION_TERMINATION_DETAILS% \"%UPSTREAM_TRANSPORT_FAILURE_REASON%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME% %ROUTE_NAME%\n"
}
}
}
]
}
]
Here’s my config.
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: proxy
spec:
selector:
istio: ingressgateway
servers:
- port:
number: 80
name: http
protocol: HTTP
hosts:
- "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
name: example
spec:
hosts:
- www.example.com
ports:
- number: 8080
name: test
protocol: HTTP
resolution: DNS
location: MESH_EXTERNAL
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: ping
spec:
hosts:
- "*"
gateways:
- proxy
http:
- match:
- gateways:
- proxy
port: 80
uri:
prefix: /
route:
- destination:
host: www.example.com
port:
number: 8080
---
apiVersion: security.istio.io/v1beta1
kind: RequestAuthentication
metadata:
name: jwt-example
spec:
jwtRules:
- issuer: "https://auth.example.com"
jwksUri: "https://auth.example.com/jwks.json"
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: frontend-ingress
spec:
action: ALLOW
rules:
- {}