Due to security requirements, I need to install Istio offline on an AWS EKS cluster. I could not find any clear documentation on offline installation or recommendations. So I have used the “istioctl generate manifest > generated-manifest.yml” to save the installation files, I have replaced the 3 images (proxy, pilot, prometheus) with my images hosted in AWS ECR and applied the file with kubectl. I should mention that “istioctl apply manifest” times out for some reason. I have also created the “istio-system” namespace manually before installation.
I have used these 2 annotations for an internal Network Load Balancer:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
The installation creates the CRDs, the pods and services. Here are my questions, if someone can point me into the right direction:
- Where can I find some proper documentation for offline installation?
- There is no istio-egressgateway with the default profile. Is the egress gateway not needed if I need to expose an application?
- The NLB is created with 4 listeners (80, 443, 15443, 15021) but I can connect only to 15201, the others refuse the connection. Hence only the 15021 target group is healthy, the other 3 are not.
Adding below some of the relevant code:
kubectl describe service/istio-ingressgateway -n istio-system
------
Name: istio-ingressgateway
Namespace: istio-system
Labels: app=istio-ingressgateway
istio=ingressgateway
release=istio
Annotations: service.beta.kubernetes.io/aws-load-balancer-internal: true
service.beta.kubernetes.io/aws-load-balancer-type: nlb
Selector: app=istio-ingressgateway,istio=ingressgateway
Type: LoadBalancer
IP Family Policy: SingleStack
IP Families: IPv4
IP: 172.20.212.217
IPs: 172.20.212.217
LoadBalancer Ingress: aaaa-bbb.elb.eu-west-2.amazonaws.com
Port: status-port 15021/TCP
TargetPort: 15021/TCP
NodePort: status-port 32094/TCP
Endpoints: 10.0.35.41:15021
Port: http2 80/TCP
TargetPort: 80/TCP
NodePort: http2 30890/TCP
Endpoints: 10.0.35.41:80
Port: https 443/TCP
TargetPort: 443/TCP
NodePort: https 32039/TCP
Endpoints: 10.0.35.41:443
Port: tls 15443/TCP
TargetPort: 15443/TCP
NodePort: tls 30690/TCP
Endpoints: 10.0.35.41:15443
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
sh-4.2$ kubectl describe service/istiod -n istio-system
Name: istiod
Namespace: istio-system
Labels: app=istiod
istio=pilot
istio.io/rev=default
release=istio
Annotations: <none>
Selector: app=istiod,istio=pilot
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 172.20.25.159
IPs: 172.20.25.159
Port: grpc-xds 15010/TCP
TargetPort: 15010/TCP
Endpoints: 10.0.43.252:15010
Port: https-dns 15012/TCP
TargetPort: 15012/TCP
Endpoints: 10.0.43.252:15012
Port: https-webhook 443/TCP
TargetPort: 15017/TCP
Endpoints: 10.0.43.252:15017
Port: http-monitoring 15014/TCP
TargetPort: 15014/TCP
Endpoints: 10.0.43.252:15014
Port: dns-tls 853/TCP
TargetPort: 15053/TCP
Endpoints: 10.0.43.252:15053
Session Affinity: None
Events: <none>
kind: CustomResourceDefinition
apiVersion: apiextensions.k8s.io/v1beta1
metadata:
name: adapters.config.istio.io
labels:
app: mixer
package: adapter
istio: mixer-adapter
chart: istio
heritage: Tiller
release: istio
annotations:
"helm.sh/resource-policy": keep
spec:
group: config.istio.io
names:
kind: adapter
plural: adapters
singular: adapter
categories:
- istio-io
- policy-istio-io
scope: Namespaced
subresources:
status: {}
versions:
- name: v1alpha2
served: true
storage: true
---
kind: CustomResourceDefinition
apiVersion: apiextensions.k8s.io/v1beta1
metadata:
name: templates.config.istio.io
labels:
app: mixer
package: template
istio: mixer-template
chart: istio
heritage: Tiller
release: istio
annotations:
"helm.sh/resource-policy": keep
spec:
group: config.istio.io
names:
kind: template
plural: templates
singular: template
categories:
- istio-io
- policy-istio-io
scope: Namespaced
subresources:
status: {}
versions:
- name: v1alpha2
served: true
storage: true
---
# Cni component is disabled.
# EgressGateways istio-egressgateway component is disabled.
# Resources for IngressGateways component
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
labels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
name: istio-ingressgateway
namespace: istio-system
spec:
maxReplicas: 5
metrics:
- resource:
name: cpu
targetAverageUtilization: 80
type: Resource
minReplicas: 1
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: istio-ingressgateway
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
name: istio-ingressgateway
namespace: istio-system
spec:
selector:
matchLabels:
app: istio-ingressgateway
istio: ingressgateway
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 25%
template:
metadata:
annotations:
sidecar.istio.io/inject: "false"
labels:
app: istio-ingressgateway
chart: gateways
heritage: Tiller
istio: ingressgateway
release: istio
service.istio.io/canonical-name: istio-ingressgateway
service.istio.io/canonical-revision: latest
spec:
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- preference:
matchExpressions:
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
weight: 2
- preference:
matchExpressions:
- key: beta.kubernetes.io/arch
operator: In
values:
- ppc64le
weight: 2
- preference:
matchExpressions:
- key: beta.kubernetes.io/arch
operator: In
values:
- s390x
weight: 2
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
- ppc64le
- s390x
containers:
- args:
- proxy
- router
- --domain
- $(POD_NAMESPACE).svc.cluster.local
- --proxyLogLevel=warning
- --proxyComponentLogLevel=misc:error
- --log_output_level=default:info
- --serviceCluster
- istio-ingressgateway
- --trust-domain=cluster.local
env:
- name: JWT_POLICY
value: third-party-jwt
- name: PILOT_CERT_PROVIDER
value: istiod
- name: CA_ADDR
value: istiod.istio-system.svc:15012
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: INSTANCE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: SERVICE_ACCOUNT
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
- name: CANONICAL_SERVICE
valueFrom:
fieldRef:
fieldPath: metadata.labels['service.istio.io/canonical-name']
- name: CANONICAL_REVISION
valueFrom:
fieldRef:
fieldPath: metadata.labels['service.istio.io/canonical-revision']
- name: ISTIO_META_WORKLOAD_NAME
value: istio-ingressgateway
- name: ISTIO_META_OWNER
value: kubernetes://apis/apps/v1/namespaces/istio-system/deployments/istio-ingressgateway
- name: ISTIO_META_MESH_ID
value: cluster.local
- name: ISTIO_META_ROUTER_MODE
value: sni-dnat
- name: ISTIO_META_CLUSTER_ID
value: Kubernetes
image: 777777777777.dkr.ecr.eu-west-2.amazonaws.com/istio-proxyv2:1.6.8
name: istio-proxy
ports:
- containerPort: 15021
- containerPort: 8080
- containerPort: 8443
- containerPort: 15443
- containerPort: 15011
- containerPort: 15012
- containerPort: 8060
- containerPort: 853
- containerPort: 15090
name: http-envoy-prom
protocol: TCP
readinessProbe:
failureThreshold: 30
httpGet:
path: /healthz/ready
port: 15021
scheme: HTTP
initialDelaySeconds: 1
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: 2000m
memory: 1024Mi
requests:
cpu: 100m
memory: 128Mi
volumeMounts:
- mountPath: /etc/istio/proxy
name: istio-envoy
- mountPath: /etc/istio/config
name: config-volume
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
- mountPath: /var/run/secrets/tokens
name: istio-token
readOnly: true
- mountPath: /var/run/ingress_gateway
name: ingressgatewaysdsudspath
- mountPath: /etc/istio/pod
name: podinfo
- mountPath: /etc/istio/ingressgateway-certs
name: ingressgateway-certs
readOnly: true
- mountPath: /etc/istio/ingressgateway-ca-certs
name: ingressgateway-ca-certs
readOnly: true
serviceAccountName: istio-ingressgateway-service-account
volumes:
- configMap:
name: istio-ca-root-cert
name: istiod-ca-cert
- downwardAPI:
items:
- fieldRef:
fieldPath: metadata.labels
path: labels
- fieldRef:
fieldPath: metadata.annotations
path: annotations
name: podinfo
- emptyDir: {}
name: istio-envoy
- emptyDir: {}
name: ingressgatewaysdsudspath
- name: istio-token
projected:
sources:
- serviceAccountToken:
audience: istio-ca
expirationSeconds: 43200
path: istio-token
- configMap:
name: istio
optional: true
name: config-volume
- name: ingressgateway-certs
secret:
optional: true
secretName: istio-ingressgateway-certs
- name: ingressgateway-ca-certs
secret:
optional: true
secretName: istio-ingressgateway-ca-certs
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: istio-ingressgateway
namespace: istio-system
labels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
spec:
minAvailable: 1
selector:
matchLabels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: istio-ingressgateway-sds
namespace: istio-system
labels:
release: istio
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: istio-ingressgateway-sds
namespace: istio-system
labels:
release: istio
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: istio-ingressgateway-sds
subjects:
- kind: ServiceAccount
name: istio-ingressgateway-service-account
---
apiVersion: v1
kind: Service
metadata:
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
labels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
name: istio-ingressgateway
namespace: istio-system
spec:
ports:
- name: status-port
port: 15021
targetPort: 15021
- name: http2
port: 80
targetPort: 80
- name: https
port: 443
targetPort: 443
- name: tls
port: 15443
targetPort: 15443
selector:
app: istio-ingressgateway
istio: ingressgateway
type: LoadBalancer
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: istio-ingressgateway-service-account
namespace: istio-system
labels:
app: istio-ingressgateway
istio: ingressgateway
release: istio
---
# IstiodRemote component is disabled.
# Resources for Pilot component
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
name: istiod
namespace: istio-system
labels:
app: istiod
release: istio
istio.io/rev: default
spec:
maxReplicas: 5
minReplicas: 1
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: istiod
metrics:
- type: Resource
resource:
name: cpu
targetAverageUtilization: 80
---