I’ve been trying to get keycloak configured to allow for ingress calls to be authenticated with JWT tokens. However, after configuring both keycloak and Istio I get continual 401’s with no errors in any logs anywhere. The closest I can get to an error is the ingressgateway pod logging the 401 was returned. I noticed I originaly didn’t include the .keycloak in the configuration and Istio would complain that it could not find the certificates. However, since fixing this issue, it still doesn’t seem that Istio is using the certificates. Am I missing something? Any pointers on where/how to debug this is greatly appreciated.
Versions:
- K8s: 1.11.6-gke.2
- Istio: 1.0.5
- Keycloak: 4.8.3.Final
I’ve gone through the troubleshooting here. But it doesn’t really tell you how to continue debugging if the certs don’t exist.
My steps are as follows:
# Install istio and helm on brand new GKE cluster
helm init \
--tiller-tls \
--tiller-tls-cert tiller.cert.pem \
--tiller-tls-key tiller.key.pem \
--tiller-tls-verify \
--tls-ca-cert ca.cert.pem \
--service-account tiller
helm install {tls stuff} install/kubernetes/helm/istio --set security.enabled=true --name istio --namespace istio-system
helm install {tls stuff} keycloak --namespace keycloak
# Configure default namespace for sidecars
k label namespace default istio-injection=enabled
# Deploy the bookinfo app
k apply -f samples/bookinfo/platform/kube/bookinfo.yaml
k apply -f samples/bookinfo/networking/bookinfo-gateway.yaml
# Export ingress
export INGRESS_HOST=$(k -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
export INGRESS_PORT=$(k -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].port}')
export SECURE_INGRESS_PORT=$(k -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].port}')
# Test Curl
curl $INGRESS_HOST/api/v1/products -s -o /dev/null -w "%{http_code}\n"
200
# Create istio realm in keycloak via UI, create user, etc
# ...
# Create the Policy:
cat <<EOF | k apply -f -
apiVersion: "authentication.istio.io/v1alpha1"
kind: "Policy"
metadata:
name: "ingressgateway"
namespace: istio-system
spec:
targets:
- name: istio-ingressgateway
origins:
- jwt:
issuer: "http://keycloak-http.keycloak/auth/realms/istio"
jwksUri: "http://keycloak-http.keycloak/auth/realms/istio/protocol/openid-connect/certs"
principalBinding: USE_ORIGIN
EOF
# Test Curl Again
curl $INGRESS_HOST/api/v1/products -s -o /dev/null -w "%{http_code}\n"
401
# Get token
k port-forward --namespace keycloak $POD_NAME 8080
TOKEN=$(curl http://127.0.0.1:8080/auth/realms/istio/protocol/openid-connect/token -d 'username=****&password=*****&grant_type=password&client_id=test'| jq -r ".access_token")
# Attempt curl again:
curl --header "Authorization: Bearer $TOKEN" $INGRESS_HOST/api/v1/products -s -o /dev/null -w "%{http_code}\n"
401
# Notice there are no certs in the ingress
k exec -it -n istio-system $(k -n istio-system get pods -l istio=ingressgateway -o jsonpath='{.items[0].metadata.name}') -- ls -al /etc/istio/ingressgateway-certs
total 8
drwxrwxrwt 3 root root 80 Jan 21 16:04 .
drwxr-xr-x 1 root root 4096 Jan 21 16:05 ..
drwxr-xr-x 2 root root 40 Jan 21 16:04 ..2019_01_21_16_04_59.194970182
lrwxrwxrwx 1 root root 31 Jan 21 16:04 ..data -> ..2019_01_21_16_04_59.194970182
# Notice the ingress is unaware of the cert chain
k exec -ti $(k get po -l istio=ingressgateway -n istio-system -o jsonpath={.items[0]..metadata.name}) -n istio-system -- curl 127.0.0.1:15000/certs
{
"ca_cert": "Certificate Path: /etc/certs/root-cert.pem, Serial Number: d21fd824708e78c89704373c6487f13a, Days until Expiration: 364",
"cert_chain": "Certificate Path: /etc/certs/cert-chain.pem, Serial Number: 0d0a1e2ea98fa68d3248c3853c4acc23, Days until Expiration: 89"
}