# Variables which should be replaced. Similar to grafana dashboards' __inputs section
x-inputs:
  - name: namespace
    type: constant
    value: zenko
  - name: service
    type: constant
    value: artesca-data-connector-s3api-metrics
  - name: reportJob
    type: constant
    value: artesca-data-ops-report-handler
  - name: replicas
    type: constant
  - name: systemErrorsWarningThreshold
    type: config
    value: 0.03
  - name: systemErrorsCriticalThreshold
    type: config
    value: 0.05
  - name: listingLatencyWarningThreshold
    type: config
    value: 0.300
  - name: listingLatencyCriticalThreshold
    type: config
    value: 0.500
  - name: deleteLatencyWarningThreshold
    type: config
    value: 0.500
  - name: deleteLatencyCriticalThreshold
    type: config
    value: 1.000
  - name: quotaUnavailabilityThreshold
    type: config
    value: 0.500

groups:
- name: CloudServer
  rules:

  - alert: DataAccessS3EndpointDegraded
    expr: sum(up{namespace="${namespace}", service="${service}"}) < ${replicas}
    for:   "30s"
    labels:
      severity: warning
    annotations:
      description: "Less than 100% of S3 endpoints are up and healthy"
      summary: "Data Access service is degraded"

  - alert: DataAccessS3EndpointCritical
    expr: sum(up{namespace="${namespace}", service="${service}"}) * 2 < ${replicas}
    for:   "30s"
    labels:
      severity: critical
    annotations:
      description: "Less than 50% of S3 endpoints are up and healthy"
      summary: "Data Access service is critical"

  # As a platform admin I want to be alerted (warning) when the system errors are more than 3% of
  # all the response codes
  - alert: SystemErrorsWarning
    expr: |
      sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m]))
          / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m]))
        >= ${systemErrorsWarningThreshold}
    for: 5m
    labels:
      severity: warning
    annotations:
      description: "System errors represent more than 3% of all the response codes"
      summary: "High ratio of system erors"

  # As a platform admin I want to be alerted (critical) when the system errors are more than 5% of
  # all the response codes
  - alert: SystemErrorsCritical
    expr: |
      sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m]))
          / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m]))
        >= ${systemErrorsCriticalThreshold}
    for: 5m
    labels:
      severity: critical
    annotations:
      description: "System errors represent more than 5% of all the response codes"
      summary: "Very high ratio of system erors"

  # As a platform admin I want to be alerted (warning) when a listing operation latency or a
  # version listing operation latency is more than 300ms
  - alert: ListingLatencyWarning
    expr: |
      sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m]))
          / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m]))
        >= ${listingLatencyWarningThreshold}
    for: 5m
    labels:
      severity: warning
    annotations:
      description: "Latency of listing or version listing operations is more than 300ms"
      summary: "High listing latency"

  # As a platform admin I want to be alerted (critical) when a listing operation latency or a
  # version listing operation latency is more than 500ms
  - alert: ListingLatencyCritical
    expr: |
      sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m]))
          / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m]))
        >= ${listingLatencyCriticalThreshold}
    for: 5m
    labels:
      severity: critical
    annotations:
      description: "Latency of listing or version listing operations is more than 500ms"
      summary: "Very high listing latency"

  # As a platform admin I want to be alerted (warning) when a delete operation latency is more than
  # 500ms
  - alert: DeleteLatencyWarning
    expr: |
      sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m]))
          / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m]))
        >= ${deleteLatencyWarningThreshold}
    for: 5m
    labels:
      severity: warning
    annotations:
      description: "Latency of delete object operations is more than 500ms"
      summary: "High delete latency"

  # As a platform admin I want to be alerted (critical) when a delete operation latency is more
  # than 1s
  - alert: DeleteLatencyCritical
    expr: |
      sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m]))
          / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m]))
        >= ${deleteLatencyCriticalThreshold}
    for: 5m
    labels:
      severity: critical
    annotations:
      description: "Latency of delete object operations is more than 1s"
      summary: "Very high delete latency"

  # As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled
  # but not available for at least half of the S3 services during the last minute
  - alert: QuotaMetricsNotAvailable
    expr: |
      avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"})
        < ${quotaUnavailabilityThreshold} and
        (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or
         max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0)
    labels:
      severity: warning
    annotations:
      description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled."
      summary: "Utilization metrics service not available"

  # As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled
  # but not available during the last 10 minutes
  - alert: QuotaMetricsNotAvailable
    expr: |
      avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"})
        < ${quotaUnavailabilityThreshold} and
        (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or
         max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0)
    for: 10m
    labels:
      severity: critical
    annotations:
      description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled."
      summary: "Utilization metrics service not available"

  # As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics
  # being unavailable
  - alert: QuotaUnavailable
    expr: |
      sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m]))
        > 0
    for: 5m
    labels:
      severity: critical
    annotations:
      description: "Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet."
      summary: "High number of quota requests with metrics unavailable"
