V-1.1 - communication outside cluster is failing

Hello Users/contributors,

In my setup we send all the calls going out of cluster to an Internal Load Balancer in GCP. We do this by creating a egress service and manually adding endpoints to this service. The endpoint to this service is the IP of the Internal load balancer.

In one of our cluster thins started working fine but it got resolved unintentionally. I reproduced this problem in another cluster but couldnt reproduce the resolution.

Working cluster egress IP - 10.206.117.116
Non-working cluster egress IP - 10.206.180.135
Internal Load Balancer IP and ports - 10.207.132.8 (30080 and 30443)

Our config looks like this:

Working cluster:


Non-Working cluster:

Working Cluster (istioctl proxy-config endpoint/route/cluster/listener):

Listener:

{
    "name": "10.206.117.116_443",
    "address": {
        "socketAddress": {
            "address": "10.206.117.116",
            "portValue": 443
        }
    },
    "filterChains": [
        {
            "filters": [
                {
                    "name": "mixer",
                    "config": {
                        "disable_check_calls": true,
                        "mixer_attributes": {
                            "attributes": {
                                "context.reporter.kind": {
                                    "string_value": "outbound"
                                },
                                "context.reporter.uid": {
                                    "string_value": "kubernetes://auth-29-f6f598f45-nvczr.r19-1-auth-qa"
                                },
                                "destination.service.host": {
                                    "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                },
                                "destination.service.name": {
                                    "string_value": "egproxy"
                                },
                                "destination.service.namespace": {
                                    "string_value": "egproxy-lle"
                                },
                                "destination.service.uid": {
                                    "string_value": "istio://egproxy-lle/services/egproxy"
                                },
                                "source.namespace": {
                                    "string_value": "r19-1-auth-qa"
                                },
                                "source.uid": {
                                    "string_value": "kubernetes://auth-29-f6f598f45-nvczr.r19-1-auth-qa"
                                }
                            }
                        },
                        "transport": {
                            "check_cluster": "outbound|9091||istio-policy.istio-system.svc.cluster.local",
                            "network_fail_policy": {
                                "base_retry_wait": "0.080s",
                                "max_retry_wait": "1s",
                                "policy": "FAIL_CLOSE"
                            },
                            "report_cluster": "outbound|9091||istio-telemetry.istio-system.svc.cluster.local"
                        }
                    }
                },
                {
                    "name": "envoy.tcp_proxy",
                    "config": {
                        "access_log": [
                            {
                                "config": {
                                    "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% \"%DYNAMIC_METADATA(istio.mixer:status)%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME%\n",
                                    "path": "/dev/stdout"
                                },
                                "name": "envoy.file_access_log"
                            }
                        ],
                        "cluster": "outbound|443||egproxy.egproxy-lle.svc.cluster.local",
                        "stat_prefix": "outbound|443||egproxy.egproxy-lle.svc.cluster.local"
                    }
                }
            ]
        }
    ],
    "deprecatedV1": {
        "bindToPort": false
    }
},

Route:

        {
            "name": "egproxy.egproxy-lle.svc.cluster.local:80",
            "domains": [
                "egproxy.egproxy-lle.svc.cluster.local",
                "egproxy.egproxy-lle.svc.cluster.local:80",
                "egproxy.egproxy-lle",
                "egproxy.egproxy-lle:80",
                "egproxy.egproxy-lle.svc.cluster",
                "egproxy.egproxy-lle.svc.cluster:80",
                "egproxy.egproxy-lle.svc",
                "egproxy.egproxy-lle.svc:80",
                "10.206.117.116",
                "10.206.117.116:80"
            ],
            "routes": [
                {
                    "match": {
                        "prefix": "/"
                    },
                    "route": {
                        "cluster": "outbound|80||egproxy.egproxy-lle.svc.cluster.local",
                        "timeout": "0s",
                        "retryPolicy": {
                            "retryOn": "connect-failure,refused-stream,unavailable,cancelled,resource-exhausted,retriable-status-codes",
                            "numRetries": 2,
                            "retryHostPredicate": [
                                {
                                    "name": "envoy.retry_host_predicates.previous_hosts"
                                }
                            ],
                            "hostSelectionRetryMaxAttempts": "3",
                            "retriableStatusCodes": [
                                503
                            ]
                        },
                        "maxGrpcTimeout": "0s"
                    },
                    "decorator": {
                        "operation": "egproxy.egproxy-lle.svc.cluster.local:80/*"
                    },
                    "perFilterConfig": {
                        "mixer": {
                            "disable_check_calls": true,
                            "forward_attributes": {
                                "attributes": {
                                    "destination.service.host": {
                                        "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                    },
                                    "destination.service.name": {
                                        "string_value": "egproxy"
                                    },
                                    "destination.service.namespace": {
                                        "string_value": "egproxy-lle"
                                    },
                                    "destination.service.uid": {
                                        "string_value": "istio://egproxy-lle/services/egproxy"
                                    }
                                }
                            },
                            "mixer_attributes": {
                                "attributes": {
                                    "destination.service.host": {
                                        "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                    },
                                    "destination.service.name": {
                                        "string_value": "egproxy"
                                    },
                                    "destination.service.namespace": {
                                        "string_value": "egproxy-lle"
                                    },
                                    "destination.service.uid": {
                                        "string_value": "istio://egproxy-lle/services/egproxy"
                                    }
                                }
                            }
                        }
                    }
                }
            ]
        },

Non-working Cluster:

Listener:

    {
    "name": "10.206.180.135_443",
    "address": {
        "socketAddress": {
            "address": "10.206.180.135",
            "portValue": 443
        }
    },
    "filterChains": [
        {
            "filterChainMatch": {
                "serverNames": [
                    "egproxy.egproxy-lle.svc.cluster.local"
                ]
            },
            "filters": [
                {
                    "name": "mixer",
                    "config": {
                        "disable_check_calls": true,
                        "mixer_attributes": {
                            "attributes": {
                                "context.reporter.kind": {
                                    "string_value": "outbound"
                                },
                                "context.reporter.uid": {
                                    "string_value": "kubernetes://auth-35-864dd85ff-rjfzs.r19-3-auth-qa"
                                },
                                "destination.service.host": {
                                    "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                },
                                "destination.service.name": {
                                    "string_value": "egproxy"
                                },
                                "destination.service.namespace": {
                                    "string_value": "egproxy-lle"
                                },
                                "destination.service.uid": {
                                    "string_value": "istio://egproxy-lle/services/egproxy"
                                },
                                "source.namespace": {
                                    "string_value": "r19-3-auth-qa"
                                },
                                "source.uid": {
                                    "string_value": "kubernetes://auth-35-864dd85ff-rjfzs.r19-3-auth-qa"
                                }
                            }
                        },
                        "transport": {
                            "check_cluster": "outbound|9091||istio-policy.istio-system.svc.cluster.local",
                            "network_fail_policy": {
                                "base_retry_wait": "0.080s",
                                "max_retry_wait": "1s",
                                "policy": "FAIL_CLOSE"
                            },
                            "report_cluster": "outbound|9091||istio-telemetry.istio-system.svc.cluster.local"
                        }
                    }
                },
                {
                    "name": "envoy.tcp_proxy",
                    "config": {
                        "access_log": [
                            {
                                "config": {
                                    "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% \"%DYNAMIC_METADATA(istio.mixer:status)%\" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME%\n",
                                    "path": "/dev/stdout"
                                },
                                "name": "envoy.file_access_log"
                            }
                        ],
                        "cluster": "outbound|443||egproxy.egproxy-lle.svc.cluster.local",
                        "stat_prefix": "outbound|443||egproxy.egproxy-lle.svc.cluster.local"
                                                }
                }
            ],
            "metadata": {
                "filterMetadata": {
                    "istio": {
                        "config": "/apis/networking/v1alpha3/namespaces/r19-3-mui-qa/virtual-service/vs-ilb"
                    }
                }
            }
        }
    ],
    "deprecatedV1": {
        "bindToPort": false
    },
    "listenerFilters": [
        {
            "name": "envoy.listener.tls_inspector"
        }
    ]
},

Route:

            {
            "name": "egproxy.egproxy-lle.svc.cluster.local:80",
            "domains": [
                "egproxy.egproxy-lle.svc.cluster.local",
                "egproxy.egproxy-lle.svc.cluster.local:80",
                "egproxy.egproxy-lle",
                "egproxy.egproxy-lle:80",
                "egproxy.egproxy-lle.svc.cluster",
                "egproxy.egproxy-lle.svc.cluster:80",
                "egproxy.egproxy-lle.svc",
                "egproxy.egproxy-lle.svc:80",
                "10.206.180.135",
                "10.206.180.135:80"
            ],
            "routes": [
                {
                    "match": {
                        "prefix": "/"
                    },
                    "route": {
                        "cluster": "outbound|80||egproxy.egproxy-lle.svc.cluster.local",
                        "timeout": "0s",
                        "retryPolicy": {
                            "retryOn": "connect-failure,refused-stream,unavailable,cancelled,resource-exhausted,retriable-status-codes",
                            "numRetries": 2,
                            "retryHostPredicate": [
                                {
                                    "name": "envoy.retry_host_predicates.previous_hosts"
                                }
                            ],
                            "hostSelectionRetryMaxAttempts": "3",
                            "retriableStatusCodes": [
                                503
                            ]
                        },
                        "maxGrpcTimeout": "0s"
                    },
                    "decorator": {
                        "operation": "egproxy.egproxy-lle.svc.cluster.local:80/*"
                    },
                    "perFilterConfig": {
                        "mixer": {
                            "disable_check_calls": true,
                            "forward_attributes": {
                                "attributes": {
                                    "destination.service.host": {
                                        "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                    },
                                    "destination.service.name": {
                                        "string_value": "egproxy"
                                    },
                                    "destination.service.namespace": {
                                        "string_value": "egproxy-lle"
                                    },
                                    "destination.service.uid": {
                                        "string_value": "istio://egproxy-lle/services/egproxy"
                                    }
                                }
                            },
                            "mixer_attributes": {
                                "attributes": {
                                    "destination.service.host": {
                                        "string_value": "egproxy.egproxy-lle.svc.cluster.local"
                                    },
                                    "destination.service.name": {
                                        "string_value": "egproxy"
                                    },
                                    "destination.service.namespace": {
                                        "string_value": "egproxy-lle"
                                    },
                                    "destination.service.uid": {
                                        "string_value": "istio://egproxy-lle/services/egproxy"
                                    }
                                }
                            }
                        }
                    }
                }
            ]
        },

Curl from working cluster:

Curl from non-working cluster:

Can someone help me identify the correct config so that I can use to make this communication work ?

Can someone help me with this ? I am still stuck at this and need to find a solution for this before I can consider it for production.

Thank You