Yeah not sure what I had misconfigured but was eventually able to get things working with TLS. For anyone else that runs into this, this is what I use for exposing our grafana via an internal ingress.
# Grafana
apiVersion: "networking.istio.io/v1alpha3"
kind: "DestinationRule"
metadata:
name: "grafana"
namespace: "istio-system"
spec:
host: grafana.istio-system.svc.cluster.local
trafficPolicy:
tls:
mode: DISABLE
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: istio-grafana-gateway
namespace: "istio-system"
annotations:
kubernetes.io/ingress.class: internal # This is for external-dns integration
spec:
selector:
istio: ilbgateway
servers:
- port:
number: 443
name: https-grafana-istio
protocol: HTTPS
hosts:
- "grafana.domain"
tls: # This is a *.domain cert
mode: SIMPLE
privateKey: /etc/istio/ingressgateway-certs/tls.key
serverCertificate: /etc/istio/ingressgateway-certs/tls.crt
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: istio-grafana
namespace: "istio-system"
spec:
hosts:
- "grafana.domain"
gateways:
- istio-grafana-gateway
http:
- route:
- destination:
port:
number: 3000
host: grafana.istio-system.svc.cluster.local