Why isn't this rate limiting policy working?

I’m running Istio 1.3 and copied / adapted this straight out of the bookinfo example. I tried to make the rate limit very conservative just to see rate limiting kick in but it just doesn’t seem to (single replica deployment with traffic rate of about 11 requests per second doesn’t cause rate limiting to kick 0 in. Any thoughts ?

$ siege -b http://cmsplavm211:31380/survive/stuff/9
.
.
HTTP/1.1 202     2.50 secs:       0 bytes ==> GET  /survive/stuff/9
HTTP/1.1 202     2.50 secs:       0 bytes ==> GET  /survive/stuff/9
HTTP/1.1 202     2.50 secs:       0 bytes ==> GET  /survive/stuff/9
HTTP/1.1 202     2.50 secs:       0 bytes ==> GET  /survive/stuff/9
HTTP/1.1 202     2.50 secs:       0 bytes ==> GET  /survive/stuff/9
^C
Lifting the server siege...
Transactions:		          75 hits
Availability:		      100.00 %
Elapsed time:		        6.42 secs
Data transferred:	        0.00 MB
Response time:		        1.73 secs
Transaction rate:	       11.68 trans/sec
Throughput:		        0.00 MB/sec
Concurrency:		       20.25
Successful transactions:          75
Failed transactions:	           0
Longest transaction:	        2.50
Shortest transaction:	        0.00

apiVersion: config.istio.io/v1alpha2
kind: handler
metadata:
  name: quotahandler
  namespace: istio-system
spec:
  compiledAdapter: memquota
  params:
    quotas:
    - name: requestcountquota.instance.istio-system
      maxAmount: 5
      validDuration: 10s
      bucketDuration: 500ms
      # The first matching override is applied.
      # A requestcount instance is checked against override dimensions.
      overrides:
      # The following override applies to 'reviews' regardless
      # of the source.
      - dimensions:
          destination: springboot-webapp
        maxAmount: 1
        validDuration: 5s
---
apiVersion: config.istio.io/v1alpha2
kind: instance
metadata:
  name: requestcountquota
  namespace: istio-system
spec:
  compiledTemplate: quota
  params:
    dimensions:
      source: request.headers["x-forwarded-for"] | "unknown"
      destination: destination.labels["app"] | destination.service.name | "unknown"
      destinationVersion: destination.labels["version"] | "unknown"
---
apiVersion: config.istio.io/v1alpha2
kind: QuotaSpec
metadata:
  name: request-count
  namespace: istio-system
spec:
  rules:
  - quotas:
    - charge: 1
      quota: requestcountquota
---
apiVersion: config.istio.io/v1alpha2
kind: QuotaSpecBinding
metadata:
  name: request-count
  namespace: istio-system
spec:
  quotaSpecs:
  - name: request-count
    namespace: istio-system
  services:
  - name: springboot-webapp
    namespace: default
  # - service: '*'  # Uncomment this to bind *all* services to request-count
---
apiVersion: config.istio.io/v1alpha2
kind: rule
metadata:
  name: quota
  namespace: istio-system
spec:
  # quota only applies if you are not logged in.
  # match: match(request.headers["cookie"], "user=*") == false
  actions:
  - handler: quotahandler
    instances:
    - requestcountquota

Some more debug

$ kubectl -n istio-system get instance requestcountquota -o yaml
apiVersion: config.istio.io/v1alpha2
kind: instance
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"config.istio.io/v1alpha2","kind":"instance","metadata":{"annotations":{},"name":"requestcountquota","namespace":"istio-system"},"spec":{"compiledTemplate":"quota","params":{"dimensions":{"destination":"destination.labels[\"app\"] | destination.service.name | \"unknown\"","destinationVersion":"destination.labels[\"version\"] | \"unknown\"","source":"request.headers[\"x-forwarded-for\"] | \"unknown\""}}}}
  creationTimestamp: "2019-09-27T21:39:23Z"
  generation: 1
  name: requestcountquota
  namespace: istio-system
  resourceVersion: "2000207"
  selfLink: /apis/config.istio.io/v1alpha2/namespaces/istio-system/instances/requestcountquota
  uid: 45518eb2-e16f-11e9-9f39-005056b030f2
spec:
  compiledTemplate: quota
  params:
    dimensions:
      destination: destination.labels["app"] | destination.service.name | "unknown"
      destinationVersion: destination.labels["version"] | "unknown"
      source: request.headers["x-forwarded-for"] | "unknown"
$ kubectl -n istio-system get handler redishandler -o yaml
Error from server (NotFound): handlers.config.istio.io "redishandler" not found
$ kubectl -n istio-system get rule quota -o yaml
apiVersion: config.istio.io/v1alpha2
kind: rule
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"config.istio.io/v1alpha2","kind":"rule","metadata":{"annotations":{},"name":"quota","namespace":"istio-system"},"spec":{"actions":[{"handler":"quotahandler","instances":["requestcountquota"]}]}}
  creationTimestamp: "2019-09-27T21:39:23Z"
  generation: 1
  name: quota
  namespace: istio-system
  resourceVersion: "2000210"
  selfLink: /apis/config.istio.io/v1alpha2/namespaces/istio-system/rules/quota
  uid: 455a5fa4-e16f-11e9-9f39-005056b030f2
spec:
  actions:
  - handler: quotahandler
    instances:
    - requestcountquota
$ kubectl -n istio-system get QuotaSpec request-count -o yaml
apiVersion: config.istio.io/v1alpha2
kind: QuotaSpec
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"config.istio.io/v1alpha2","kind":"QuotaSpec","metadata":{"annotations":{},"name":"request-count","namespace":"istio-system"},"spec":{"rules":[{"quotas":[{"charge":1,"quota":"requestcountquota"}]}]}}
  creationTimestamp: "2019-09-27T21:39:23Z"
  generation: 1
  name: request-count
  namespace: istio-system
  resourceVersion: "2000208"
  selfLink: /apis/config.istio.io/v1alpha2/namespaces/istio-system/quotaspecs/request-count
  uid: 4552b5b8-e16f-11e9-9f39-005056b030f2
spec:
  rules:
  - quotas:
    - charge: 1
      quota: requestcountquota
$ kubectl -n istio-system get QuotaSpecBinding request-count -o yaml
apiVersion: config.istio.io/v1alpha2
kind: QuotaSpecBinding
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"config.istio.io/v1alpha2","kind":"QuotaSpecBinding","metadata":{"annotations":{},"name":"request-count","namespace":"istio-system"},"spec":{"quotaSpecs":[{"name":"request-count","namespace":"istio-system"}],"services":[{"name":"springboot-webapp","namespace":"default"},{"service":"*"}]}}
  creationTimestamp: "2019-09-27T21:39:23Z"
  generation: 2
  name: request-count
  namespace: istio-system
  resourceVersion: "2001820"
  selfLink: /apis/config.istio.io/v1alpha2/namespaces/istio-system/quotaspecbindings/request-count
  uid: 4557b770-e16f-11e9-9f39-005056b030f2
spec:
  quotaSpecs:
  - name: request-count
    namespace: istio-system
  services:
  - name: springboot-webapp
    namespace: default
  - service: '*'

$ kc get svc
NAME                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
httpbin             ClusterIP   10.43.238.71    <none>        80/TCP     11d
kubernetes          ClusterIP   10.43.0.1       <none>        443/TCP    14d
springboot-webapp   ClusterIP   10.43.113.160   <none>        7070/TCP   14d

$ kc get vs springboot-webapp-virtual-service -o yaml
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"networking.istio.io/v1alpha3","kind":"VirtualService","metadata":{"annotations":{},"name":"springboot-webapp-virtual-service","namespace":"default"},"spec":{"gateways":["my-istio-gateway"],"hosts":["*"],"http":[{"match":[{"uri":{"prefix":"/survive"}},{"uri":{"prefix":"/compute"}},{"uri":{"prefix":"/greet"}},{"uri":{"prefix":"/metrics"}},{"uri":{"prefix":"/actuator"}},{"uri":{"prefix":"/fail"}}],"route":[{"destination":{"host":"springboot-webapp","port":{"number":7070}}}]}]}}
  creationTimestamp: "2019-09-20T20:29:56Z"
  generation: 5
  name: springboot-webapp-virtual-service
  namespace: default
  resourceVersion: "1986700"
  selfLink: /apis/networking.istio.io/v1alpha3/namespaces/default/virtualservices/springboot-webapp-virtual-service
  uid: 6908497f-dbe5-11e9-a02e-005056b030f2
spec:
  gateways:
  - my-istio-gateway
  hosts:
  - '*'
  http:
  - match:
    - uri:
        prefix: /survive
    - uri:
        prefix: /compute
    - uri:
        prefix: /greet
    - uri:
        prefix: /metrics
    - uri:
        prefix: /actuator
    - uri:
        prefix: /fail
    route:
    - destination:
        host: springboot-webapp
        port:
          number: 7070

Solved. Had to enable “Policy Enforcement”