I would like to understand how does Prometheus/Datadog agent collect metrics from envoy proxies? Do they use any one of below options or something else?
I tried 2 options as below and wondering why in option 2 I do not see all istio_requests_total metrics for gateway-service.
Setup : gateway-service and test-service-1 in sandbox1 (k8s namespace) and metrics-collector in sandbox2. I initiated few http requests from test-service-1 to gateway-service and metrics-collector to gateway-service and trying to collect istio_requests_total counter.
Option 1:
kubectl exec -it -n sandbox1 deploy/gateway-service -c istio-proxy – curl http://localhost:15090/stats/prometheus | grep istio_requests_total
TYPE istio_requests_total counter
istio_requests_total{response_code=“200”,reporter=“destination”, source_workload=“metrics-collector”, source_workload_namespace=“sandbox2”,source_principal=“unknown”,source_app=“metrics-collector”,source_version=“v1”,source_cluster=“Kubernetes”,destination_workload=“gateway-service”,destination_workload_namespace=“sandbox1”,destination_principal=“unknown”,destination_app=“gateway-service”,destination_version=“v1”, destination_service="gateway-service.sandbox1.svc.cluster.local “,destination_service_name=“gateway-service”,destination_service_namespace=“sandbox1”,destination_cluster=“Kubernetes”,request_protocol=“http”,response_flags=”-",grpc_response_status="",connection_security_policy=“none”,source_canonical_service=“metrics-collector”,destination_canonical_service=“gateway-service”,source_canonical_revision=“v1”,destination_canonical_revision=“v1”} 2
istio_requests_total{response_code=“200”,reporter=“destination”, source_workload=“test-service-3” ,source_workload_namespace=“sandbox1”,source_principal=“unknown”,source_app=“test-service-3”,source_version=“v1”,source_cluster=“Kubernetes”,destination_workload=“gateway-service”,destination_workload_namespace=“sandbox1”,destination_principal=“unknown”,destination_app=“gateway-service”,destination_version=“v1”, destination_service=“gateway-service.sandbox1.svc.cluster.local” ,destination_service_name=“gateway-service”,destination_service_namespace=“sandbox1”,destination_cluster=“Kubernetes”,request_protocol=“http”,response_flags="-",grpc_response_status="",connection_security_policy=“none”,source_canonical_service=“test-service-3”,destination_canonical_service=“gateway-service”,source_canonical_revision=“v1”,destination_canonical_revision=“v1”} 10
Option 2:
kubectl exec -it -n sandbox2 deploy/metrics-collector – curl http://gateway-service.sandbox1:15090/stats/prometheus | grep istio_requests_total
TYPE istio_requests_total counter
istio_requests_total{response_code=“200”,reporter=“source”, source_workload=“metrics-collector”, source_workload_namespace=“sandbox2”,source_principal=“unknown”,source_app=“metrics-collector”,source_version=“v1”,source_cluster=“Kubernetes”,destination_workload=“gateway-service”,destination_workload_namespace=“sandbox1”,destination_principal=“unknown”,destination_app=“gateway-service”,destination_version=“v1”, destination_service=“gateway-service.sandbox1.svc.cluster.local” ,destination_service_name=“gateway-service”,destination_service_namespace=“sandbox1”,destination_cluster=“Kubernetes”,request_protocol=“http”,response_flags="-",grpc_response_status="",connection_security_policy=“unknown”,source_canonical_service=“metrics-collector”,destination_canonical_service=“gateway-service”,source_canonical_revision=“v1”,destination_canonical_revision=“v1”} 1
istio_requests_total{response_code=“200”,reporter=“source”,source_workload=“metrics-collector”,source_workload_namespace=“sandbox2”,source_principal=“unknown”,source_app=“metrics-collector”,source_version=“v1”,source_cluster=“Kubernetes”,destination_workload=“unknown”,destination_workload_namespace=“unknown”,destination_principal=“unknown”,destination_app=“unknown”,destination_version=“unknown”,destination_service=“archive.ubuntu.com”,destination_service_name=“PassthroughCluster”,destination_service_namespace=“unknown”,destination_cluster=“unknown”,request_protocol=“http”,response_flags="-",grpc_response_status="",connection_security_policy=“unknown”,source_canonical_service=“metrics-collector”,destination_canonical_service=“unknown”,source_canonical_revision=“v1”,destination_canonical_revision=“latest”} 42
istio_requests_total{response_code=“200”,reporter=“source”,source_workload=“metrics-collector”,source_workload_namespace=“sandbox2”,source_principal=“unknown”,source_app=“metrics-collector”,source_version=“v1”,source_cluster=“Kubernetes”,destination_workload=“unknown”,destination_workload_namespace=“unknown”,destination_principal=“unknown”,destination_app=“unknown”,destination_version=“unknown”,destination_service=“security.ubuntu.com”,destination_service_name=“PassthroughCluster”,destination_service_namespace=“unknown”,destination_cluster=“unknown”,request_protocol=“http”,response_flags="-",grpc_response_status="",connection_security_policy=“unknown”,source_canonical_service=“metrics-collector”,destination_canonical_service=“unknown”,source_canonical_revision=“v1”,destination_canonical_revision=“latest”} 5