Hi All,
I tried to integrate VMs into Istio mesh(v1.2) deployed on K8s, and the node agent starts successfully, but the istio service failed to start. Any ideas on this? Thanks.
Here is the logs:
root@sdc-xxxxx:~/istio# systemctl status istio.service
● istio.service - istio-sidecar: The Istio sidecar
Loaded: loaded (/lib/systemd/system/istio.service; disabled; vendor preset: enabled)
Active: activating (auto-restart) (Result: exit-code) since Tue 2019-06-25 02:37:14 PDT; 621ms ago
Docs: http://istio.io/
Process: 23304 ExecStart=/usr/local/bin/istio-start.sh (code=exited, status=2)
Main PID: 23304 (code=exited, status=2)
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: -A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: -A DOCKER-ISOLATION-STAGE-2 -o br-fed57d587b34 -j DROP
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: -A DOCKER-ISOLATION-STAGE-2 -j RETURN
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: -A DOCKER-USER -j RETURN
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: COMMIT
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: # Completed on Tue Jun 25 02:37:14 2019
Jun 25 02:37:14 sdc-xxxxx istio-start.sh[23304]: + ip6tables-save
Jun 25 02:37:14 sdc-xxxxx systemd[1]: istio.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
Jun 25 02:37:14 sdc-xxxxx systemd[1]: istio.service: Unit entered failed state.
Jun 25 02:37:14 sdc-xxxxx systemd[1]: istio.service: Failed with result 'exit-code'.
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: Environment:
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ------------
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ENVOY_PORT=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_INBOUND_INTERCEPTION_MODE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_INBOUND_TPROXY_MARK=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_INBOUND_TPROXY_ROUTE_TABLE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_INBOUND_PORTS=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_LOCAL_EXCLUDE_PORTS=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_SERVICE_CIDR=10.1.0.0/16
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ISTIO_SERVICE_EXCLUDE_CIDR=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: Variables:
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ----------
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: PROXY_PORT=15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_CAPTURE_PORT=15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: PROXY_UID=115,0
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_INTERCEPTION_MODE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_TPROXY_MARK=1337
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_TPROXY_ROUTE_TABLE=133
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_PORTS_INCLUDE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: INBOUND_PORTS_EXCLUDE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: OUTBOUND_IP_RANGES_INCLUDE=10.1.0.0/16
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: OUTBOUND_IP_RANGES_EXCLUDE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: OUTBOUND_PORTS_EXCLUDE=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: KUBEVIRT_INTERFACES=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: ENABLE_INBOUND_IPV6=
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -N ISTIO_REDIRECT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-port 15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -N ISTIO_IN_REDIRECT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' -n '' ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -N ISTIO_OUTPUT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A OUTPUT -p tcp -j ISTIO_OUTPUT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' -n '' ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' -z '' ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -o lo '!' -d 127.0.0.1/32 -j ISTIO_REDIRECT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + for uid in '${PROXY_UID}'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -m owner --uid-owner 115 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + for uid in '${PROXY_UID}'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -m owner --uid-owner 0 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + for gid in '${PROXY_GID}'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -m owner --gid-owner 115 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + for gid in '${PROXY_GID}'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -m owner --gid-owner 0 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' 0 -gt 0 ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' 1 -gt 0 ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + '[' $'10.1.0.0/16\r' == '*' ']'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + for cidr in '"${ipv4_ranges_include[@]}"'
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables -t nat -A ISTIO_OUTPUT -d $'10.1.0.0/16\r' -j ISTIO_REDIRECT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: [45B blob data]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: Try `iptables -h' or 'iptables --help' for more information.
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + dump
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + iptables-save
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Generated by iptables-save v1.6.0 on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: *mangle
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :PREROUTING ACCEPT [1927766:176148706]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :INPUT ACCEPT [394316:51991555]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :FORWARD ACCEPT [30014:4665010]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :OUTPUT ACCEPT [37125:12426572]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :POSTROUTING ACCEPT [67139:17091582]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: COMMIT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Completed on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Generated by iptables-save v1.6.0 on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: *nat
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :PREROUTING ACCEPT [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :INPUT ACCEPT [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :OUTPUT ACCEPT [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :POSTROUTING ACCEPT [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :DOCKER - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :ISTIO_IN_REDIRECT - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :ISTIO_OUTPUT - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :ISTIO_REDIRECT - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A OUTPUT -p tcp -j ISTIO_OUTPUT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.0/16 ! -o br-fed57d587b34 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.5/32 -d 172.18.0.5/32 -p tcp -m tcp --dport 15672 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.3/32 -d 172.18.0.3/32 -p tcp -m tcp --dport 8989 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.8/32 -d 172.18.0.8/32 -p tcp -m tcp --dport 8080 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.6/32 -d 172.18.0.6/32 -p tcp -m tcp --dport 8761 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A POSTROUTING -s 172.18.0.7/32 -d 172.18.0.7/32 -p tcp -m tcp --dport 4000 -j MASQUERADE
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -i docker0 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -i br-fed57d587b34 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER ! -i br-fed57d587b34 -p tcp -m tcp --dport 15672 -j DNAT --to-destination 172.18.0.5:15672
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER ! -i br-fed57d587b34 -p tcp -m tcp --dport 8989 -j DNAT --to-destination 172.18.0.3:8989
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER ! -i br-fed57d587b34 -p tcp -m tcp --dport 9000 -j DNAT --to-destination 172.18.0.8:8080
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER ! -i br-fed57d587b34 -p tcp -m tcp --dport 8761 -j DNAT --to-destination 172.18.0.6:8761
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER ! -i br-fed57d587b34 -p tcp -m tcp --dport 80 -j DNAT --to-destination 172.18.0.7:4000
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-ports 15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT ! -d 127.0.0.1/32 -o lo -j ISTIO_REDIRECT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT -m owner --uid-owner 115 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT -m owner --uid-owner 0 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT -m owner --gid-owner 115 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT -m owner --gid-owner 0 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-ports 15001
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: COMMIT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Completed on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Generated by iptables-save v1.6.0 on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: *filter
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :INPUT ACCEPT [401176:52950679]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :FORWARD DROP [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :OUTPUT ACCEPT [37978:12642463]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :DOCKER - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :DOCKER-ISOLATION-STAGE-1 - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :DOCKER-ISOLATION-STAGE-2 - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: :DOCKER-USER - [0:0]
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -j DOCKER-USER
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -j DOCKER-ISOLATION-STAGE-1
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -o docker0 -j DOCKER
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -i docker0 ! -o docker0 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -i docker0 -o docker0 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -o br-fed57d587b34 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -o br-fed57d587b34 -j DOCKER
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -i br-fed57d587b34 ! -o br-fed57d587b34 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A FORWARD -i br-fed57d587b34 -o br-fed57d587b34 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -d 172.18.0.5/32 ! -i br-fed57d587b34 -o br-fed57d587b34 -p tcp -m tcp --dport 15672 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -d 172.18.0.3/32 ! -i br-fed57d587b34 -o br-fed57d587b34 -p tcp -m tcp --dport 8989 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -d 172.18.0.8/32 ! -i br-fed57d587b34 -o br-fed57d587b34 -p tcp -m tcp --dport 8080 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -d 172.18.0.6/32 ! -i br-fed57d587b34 -o br-fed57d587b34 -p tcp -m tcp --dport 8761 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER -d 172.18.0.7/32 ! -i br-fed57d587b34 -o br-fed57d587b34 -p tcp -m tcp --dport 4000 -j ACCEPT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-1 -i br-fed57d587b34 ! -o br-fed57d587b34 -j DOCKER-ISOLATION-STAGE-2
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-1 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-2 -o br-fed57d587b34 -j DROP
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-ISOLATION-STAGE-2 -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: -A DOCKER-USER -j RETURN
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: COMMIT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: # Completed on Tue Jun 25 02:19:28 2019
Jun 25 02:19:28 sdc-xxxxx systemd[1]: istio.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
Jun 25 02:19:28 sdc-xxxxx istio-start.sh[16060]: + ip6tables-save
Jun 25 02:19:28 sdc-xxxxx systemd[1]: istio.service: Unit entered failed state.
Jun 25 02:19:28 sdc-xxxxx systemd[1]: istio.service: Failed with result 'exit-code'.
Jun 25 02:19:38 sdc-xxxxx systemd[1]: istio.service: Service hold-off time over, scheduling restart.
Jun 25 02:19:38 sdc-xxxxx systemd[1]: Stopped istio-sidecar: The Istio sidecar.
Jun 25 02:19:38 sdc-xxxxx systemd[1]: Started istio-sidecar: The Istio sidecar.