Hello Users/contributors,
In my setup we send all the calls going out of cluster to an Internal Load Balancer in GCP. We do this by creating a egress service and manually adding endpoints to this service. The endpoint to this service is the IP of the Internal load balancer.
In one of our cluster thins started working fine but it got resolved unintentionally. I reproduced this problem in another cluster but couldnt reproduce the resolution.
Working cluster egress IP - 10.206.117.116
Non-working cluster egress IP - 10.206.180.135
Internal Load Balancer IP and ports - 10.207.132.8 (30080 and 30443)
Our config looks like this:
Working cluster:
Non-Working cluster:
Working Cluster (istioctl proxy-config endpoint/route/cluster/listener):
Listener:
{ "name": "10.206.117.116_443", "address": { "socketAddress": { "address": "10.206.117.116", "portValue": 443 } }, "filterChains": [ { "filters": [ { "name": "mixer", "config": { "disable_check_calls": true, "mixer_attributes": { "attributes": { "context.reporter.kind": { "string_value": "outbound" }, "context.reporter.uid": { "string_value": "kubernetes://auth-29-f6f598f45-nvczr.r19-1-auth-qa" }, "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" }, "source.namespace": { "string_value": "r19-1-auth-qa" }, "source.uid": { "string_value": "kubernetes://auth-29-f6f598f45-nvczr.r19-1-auth-qa" } } }, "transport": { "check_cluster": "outbound|9091||istio-policy.istio-system.svc.cluster.local", "network_fail_policy": { "base_retry_wait": "0.080s", "max_retry_wait": "1s", "policy": "FAIL_CLOSE" }, "report_cluster": "outbound|9091||istio-telemetry.istio-system.svc.cluster.local" } } }, { "name": "envoy.tcp_proxy", "config": { "access_log": [ { "config": { "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% \"%DYNAMIC_METADATA(istio.mixer:status)%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME%\n", "path": "/dev/stdout" }, "name": "envoy.file_access_log" } ], "cluster": "outbound|443||egproxy.egproxy-lle.svc.cluster.local", "stat_prefix": "outbound|443||egproxy.egproxy-lle.svc.cluster.local" } } ] } ], "deprecatedV1": { "bindToPort": false } },
Route:
{ "name": "egproxy.egproxy-lle.svc.cluster.local:80", "domains": [ "egproxy.egproxy-lle.svc.cluster.local", "egproxy.egproxy-lle.svc.cluster.local:80", "egproxy.egproxy-lle", "egproxy.egproxy-lle:80", "egproxy.egproxy-lle.svc.cluster", "egproxy.egproxy-lle.svc.cluster:80", "egproxy.egproxy-lle.svc", "egproxy.egproxy-lle.svc:80", "10.206.117.116", "10.206.117.116:80" ], "routes": [ { "match": { "prefix": "/" }, "route": { "cluster": "outbound|80||egproxy.egproxy-lle.svc.cluster.local", "timeout": "0s", "retryPolicy": { "retryOn": "connect-failure,refused-stream,unavailable,cancelled,resource-exhausted,retriable-status-codes", "numRetries": 2, "retryHostPredicate": [ { "name": "envoy.retry_host_predicates.previous_hosts" } ], "hostSelectionRetryMaxAttempts": "3", "retriableStatusCodes": [ 503 ] }, "maxGrpcTimeout": "0s" }, "decorator": { "operation": "egproxy.egproxy-lle.svc.cluster.local:80/*" }, "perFilterConfig": { "mixer": { "disable_check_calls": true, "forward_attributes": { "attributes": { "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" } } }, "mixer_attributes": { "attributes": { "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" } } } } } } ] },
Non-working Cluster:
Listener:
{ "name": "10.206.180.135_443", "address": { "socketAddress": { "address": "10.206.180.135", "portValue": 443 } }, "filterChains": [ { "filterChainMatch": { "serverNames": [ "egproxy.egproxy-lle.svc.cluster.local" ] }, "filters": [ { "name": "mixer", "config": { "disable_check_calls": true, "mixer_attributes": { "attributes": { "context.reporter.kind": { "string_value": "outbound" }, "context.reporter.uid": { "string_value": "kubernetes://auth-35-864dd85ff-rjfzs.r19-3-auth-qa" }, "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" }, "source.namespace": { "string_value": "r19-3-auth-qa" }, "source.uid": { "string_value": "kubernetes://auth-35-864dd85ff-rjfzs.r19-3-auth-qa" } } }, "transport": { "check_cluster": "outbound|9091||istio-policy.istio-system.svc.cluster.local", "network_fail_policy": { "base_retry_wait": "0.080s", "max_retry_wait": "1s", "policy": "FAIL_CLOSE" }, "report_cluster": "outbound|9091||istio-telemetry.istio-system.svc.cluster.local" } } }, { "name": "envoy.tcp_proxy", "config": { "access_log": [ { "config": { "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% \"%DYNAMIC_METADATA(istio.mixer:status)%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME%\n", "path": "/dev/stdout" }, "name": "envoy.file_access_log" } ], "cluster": "outbound|443||egproxy.egproxy-lle.svc.cluster.local", "stat_prefix": "outbound|443||egproxy.egproxy-lle.svc.cluster.local" } } ], "metadata": { "filterMetadata": { "istio": { "config": "/apis/networking/v1alpha3/namespaces/r19-3-mui-qa/virtual-service/vs-ilb" } } } } ], "deprecatedV1": { "bindToPort": false }, "listenerFilters": [ { "name": "envoy.listener.tls_inspector" } ] },
Route:
{ "name": "egproxy.egproxy-lle.svc.cluster.local:80", "domains": [ "egproxy.egproxy-lle.svc.cluster.local", "egproxy.egproxy-lle.svc.cluster.local:80", "egproxy.egproxy-lle", "egproxy.egproxy-lle:80", "egproxy.egproxy-lle.svc.cluster", "egproxy.egproxy-lle.svc.cluster:80", "egproxy.egproxy-lle.svc", "egproxy.egproxy-lle.svc:80", "10.206.180.135", "10.206.180.135:80" ], "routes": [ { "match": { "prefix": "/" }, "route": { "cluster": "outbound|80||egproxy.egproxy-lle.svc.cluster.local", "timeout": "0s", "retryPolicy": { "retryOn": "connect-failure,refused-stream,unavailable,cancelled,resource-exhausted,retriable-status-codes", "numRetries": 2, "retryHostPredicate": [ { "name": "envoy.retry_host_predicates.previous_hosts" } ], "hostSelectionRetryMaxAttempts": "3", "retriableStatusCodes": [ 503 ] }, "maxGrpcTimeout": "0s" }, "decorator": { "operation": "egproxy.egproxy-lle.svc.cluster.local:80/*" }, "perFilterConfig": { "mixer": { "disable_check_calls": true, "forward_attributes": { "attributes": { "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" } } }, "mixer_attributes": { "attributes": { "destination.service.host": { "string_value": "egproxy.egproxy-lle.svc.cluster.local" }, "destination.service.name": { "string_value": "egproxy" }, "destination.service.namespace": { "string_value": "egproxy-lle" }, "destination.service.uid": { "string_value": "istio://egproxy-lle/services/egproxy" } } } } } } ] },
Curl from working cluster:
Curl from non-working cluster:
Can someone help me identify the correct config so that I can use to make this communication work ?