From 10b01fdc40ae5316c0aeeef35ac97f511f89322f Mon Sep 17 00:00:00 2001 From: plm Date: Mon, 16 Dec 2024 14:55:43 +0100 Subject: [PATCH] Adding dependencies, binary autostart --- opencloud/charts/grafana/.helmignore | 23 + opencloud/charts/grafana/Chart.yaml | 35 + opencloud/charts/grafana/README.md | 783 +++ .../charts/grafana/ci/default-values.yaml | 1 + .../grafana/ci/with-affinity-values.yaml | 16 + .../ci/with-dashboard-json-values.yaml | 53 + .../grafana/ci/with-dashboard-values.yaml | 19 + .../ci/with-extraconfigmapmounts-values.yaml | 7 + .../ci/with-image-renderer-values.yaml | 107 + .../grafana/ci/with-nondefault-values.yaml | 32 + .../charts/grafana/ci/with-persistence.yaml | 3 + .../ci/with-sidecars-envvaluefrom-values.yaml | 38 + .../grafana/dashboards/custom-dashboard.json | 1 + opencloud/charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 176 + .../charts/grafana/templates/_helpers.tpl | 274 + opencloud/charts/grafana/templates/_pod.tpl | 1389 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 20 + .../templates/dashboards-json-configmap.yaml | 35 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + opencloud/charts/grafana/templates/hpa.yaml | 51 + .../templates/image-renderer-deployment.yaml | 199 + .../grafana/templates/image-renderer-hpa.yaml | 46 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 49 + opencloud/charts/grafana/templates/pvc.yaml | 39 + opencloud/charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + opencloud/charts/grafana/templates/route.yaml | 44 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 67 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 52 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + opencloud/charts/grafana/values.yaml | 1545 +++++ opencloud/charts/loki/.helmignore | 32 + opencloud/charts/loki/Chart.lock | 12 + opencloud/charts/loki/Chart.yaml | 32 + opencloud/charts/loki/Makefile | 7 + opencloud/charts/loki/README.md | 65 + .../charts/grafana-agent-operator/.helmignore | 22 + .../charts/grafana-agent-operator/Chart.yaml | 13 + .../charts/grafana-agent-operator/README.md | 82 + .../grafana-agent-operator/README.md.gotmpl | 52 + .../monitoring.coreos.com_podmonitors.yaml | 424 ++ .../crds/monitoring.coreos.com_probes.yaml | 458 ++ ...monitoring.coreos.com_servicemonitors.yaml | 436 ++ .../monitoring.grafana.com_grafanaagents.yaml | 3711 ++++++++++ .../monitoring.grafana.com_integrations.yaml | 810 +++ .../monitoring.grafana.com_logsinstances.yaml | 299 + ...nitoring.grafana.com_metricsinstances.yaml | 495 ++ .../crds/monitoring.grafana.com_podlogs.yaml | 308 + .../templates/_helpers.tpl | 70 + .../templates/operator-clusterrole.yaml | 62 + .../operator-clusterrolebinding.yaml | 17 + .../templates/operator-deployment.yaml | 79 + .../templates/operator-serviceaccount.yaml | 10 + .../templates/tests/test-grafanaagent.yaml | 118 + .../charts/grafana-agent-operator/values.yaml | 84 + .../charts/loki/charts/minio/.helmignore | 23 + opencloud/charts/loki/charts/minio/Chart.yaml | 18 + opencloud/charts/loki/charts/minio/README.md | 264 + .../loki/charts/minio/templates/NOTES.txt | 43 + .../minio/templates/_helper_create_bucket.txt | 122 + .../minio/templates/_helper_create_policy.txt | 75 + .../templates/_helper_create_svcacct.txt | 106 + .../minio/templates/_helper_create_user.txt | 107 + .../templates/_helper_custom_command.txt | 58 + .../charts/minio/templates/_helper_policy.tpl | 28 + .../loki/charts/minio/templates/_helpers.tpl | 218 + .../minio/templates/ciliumnetworkpolicy.yaml | 33 + .../charts/minio/templates/configmap.yaml | 32 + .../minio/templates/console-ingress.yaml | 55 + .../minio/templates/console-service.yaml | 45 + .../charts/minio/templates/deployment.yaml | 213 + .../loki/charts/minio/templates/ingress.yaml | 55 + .../charts/minio/templates/networkpolicy.yaml | 26 + .../minio/templates/poddisruptionbudget.yaml | 17 + .../loki/charts/minio/templates/post-job.yaml | 258 + .../loki/charts/minio/templates/pvc.yaml | 32 + .../loki/charts/minio/templates/secrets.yaml | 21 + .../templates/securitycontextconstraints.yaml | 45 + .../loki/charts/minio/templates/service.yaml | 46 + .../minio/templates/serviceaccount.yaml | 6 + .../minio/templates/servicemonitor.yaml | 112 + .../charts/minio/templates/statefulset.yaml | 267 + .../charts/loki/charts/minio/values.yaml | 593 ++ .../loki/charts/rollout-operator/.helmignore | 23 + .../loki/charts/rollout-operator/Chart.yaml | 8 + .../loki/charts/rollout-operator/README.md | 72 + .../charts/rollout-operator/README.md.gotmpl | 38 + .../rollout-operator/templates/NOTES.txt | 10 + .../rollout-operator/templates/_helpers.tpl | 82 + .../templates/deployment.yaml | 74 + .../rollout-operator/templates/role.yaml | 30 + .../templates/rolebinding.yaml | 13 + .../rollout-operator/templates/service.yaml | 18 + .../templates/serviceaccount.yaml | 12 + .../templates/servicemonitor.yaml | 36 + .../loki/charts/rollout-operator/values.yaml | 89 + opencloud/charts/loki/distributed-values.yaml | 71 + opencloud/charts/loki/docs/examples/README.md | 4 + .../loki/docs/examples/enterprise/README.md | 28 + .../enterprise/enterprise-secrets.yaml | 12 + .../enterprise/overrides-enterprise-gcs.yaml | 83 + .../charts/loki/docs/examples/oss/README.md | 20 + .../loki/docs/examples/oss/oss-secrets.yaml | 10 + .../docs/examples/oss/overrides-oss-gcs.yaml | 77 + opencloud/charts/loki/reference.md.gotmpl | 49 + opencloud/charts/loki/scenarios/README.md | 19 + .../default-single-binary-values.yaml | 71 + .../charts/loki/scenarios/default-values.yaml | 16 + .../charts/loki/scenarios/images/added.png | Bin 0 -> 91760 bytes .../charts/loki/scenarios/images/img.png | Bin 0 -> 39609 bytes .../charts/loki/scenarios/images/modified.png | Bin 0 -> 80274 bytes .../charts/loki/scenarios/images/removed.png | Bin 0 -> 71762 bytes .../charts/loki/scenarios/ingress-values.yaml | 30 + .../scenarios/legacy-monitoring-values.yaml | 27 + .../simple-scalable-aws-kube-irsa-values.yaml | 67 + .../charts/loki/simple-scalable-values.yaml | 63 + .../charts/loki/single-binary-values.yaml | 79 + opencloud/charts/loki/src/.yamllint.yaml | 4 + opencloud/charts/loki/src/alerts.yaml.tpl | 78 + .../loki/src/dashboards/loki-chunks.json | 1336 ++++ .../loki/src/dashboards/loki-deletion.json | 632 ++ .../charts/loki/src/dashboards/loki-logs.json | 1073 +++ .../loki-mixin-recording-rules.json | 657 ++ .../loki/src/dashboards/loki-operational.json | 6173 +++++++++++++++++ .../src/dashboards/loki-reads-resources.json | 964 +++ .../loki/src/dashboards/loki-reads.json | 504 ++ .../loki/src/dashboards/loki-retention.json | 1537 ++++ .../src/dashboards/loki-writes-resources.json | 700 ++ .../loki/src/dashboards/loki-writes.json | 504 ++ .../charts/loki/src/helm-test/Dockerfile | 13 + opencloud/charts/loki/src/helm-test/README.md | 7 + .../charts/loki/src/helm-test/canary_test.go | 178 + .../charts/loki/src/helm-test/default.nix | 27 + opencloud/charts/loki/src/rules.yaml.tpl | 86 + opencloud/charts/loki/templates/NOTES.txt | 184 + opencloud/charts/loki/templates/_helpers.tpl | 1132 +++ .../loki/templates/admin-api/_helpers.yaml | 24 + .../admin-api/deployment-admin-api.yaml | 176 + .../admin-api/service-admin-api.yaml | 28 + .../templates/backend/_helpers-backend.tpl | 32 + .../loki/templates/backend/clusterrole.yaml | 20 + .../templates/backend/clusterrolebinding.yaml | 25 + .../charts/loki/templates/backend/hpa.yaml | 50 + .../backend/poddisruptionbudget-backend.yaml | 15 + .../backend/query-scheduler-discovery.yaml | 34 + .../backend/service-backend-headless.yaml | 40 + .../templates/backend/service-backend.yaml | 37 + .../backend/statefulset-backend.yaml | 287 + .../bloom-builder/_helpers-bloom-builder.tpl | 32 + .../deployment-bloom-builder.yaml | 150 + .../loki/templates/bloom-builder/hpa.yaml | 55 + .../poddisruptionbudget-bloom-builder.yaml | 21 + .../service-bloom-builder-headless.yaml | 46 + .../bloom-builder/service-bloom-builder.yaml | 44 + .../bloom-gateway/_helpers-bloom-gateway.tpl | 58 + .../service-bloom-gateway-headless.yaml | 39 + .../statefulset-bloom-gateway.yaml | 181 + .../bloom-planner/_helpers-bloom-planner.tpl | 58 + .../service-bloom-planner-headless.yaml | 37 + .../statefulset-bloom-planner.yaml | 181 + .../poddisruptionbudget-chunks-cache.yaml | 16 + .../service-chunks-cache-headless.yaml | 1 + .../statefulset-chunks-cache.yaml | 1 + .../loki/templates/ciliumnetworkpolicy.yaml | 238 + .../compactor/_helpers-compactor.tpl | 81 + .../compactor/service-compactor.yaml | 38 + .../compactor/statefulset-compactor.yaml | 193 + opencloud/charts/loki/templates/config.yaml | 21 + .../distributor/_helpers-distributor.tpl | 32 + .../distributor/deployment-distributor.yaml | 158 + .../loki/templates/distributor/hpa.yaml | 54 + .../poddisruptionbudget-distributor.yaml | 21 + .../service-distributor-headless.yaml | 39 + .../distributor/service-distributor.yaml | 36 + .../loki/templates/extra-manifests.yaml | 8 + .../templates/gateway/_helpers-gateway.tpl | 47 + .../templates/gateway/configmap-gateway.yaml | 12 + .../deployment-gateway-enterprise.yaml | 152 + .../gateway/deployment-gateway-nginx.yaml | 138 + .../charts/loki/templates/gateway/hpa.yaml | 50 + .../templates/gateway/ingress-gateway.yaml | 59 + .../gateway/poddisruptionbudget-gateway.yaml | 19 + .../templates/gateway/secret-gateway.yaml | 14 + .../templates/gateway/service-gateway.yaml | 40 + .../index-gateway/_helpers-index-gateway.tpl | 40 + .../poddisruptionbudget-index-gateway.yaml | 21 + .../service-index-gateway-headless.yaml | 35 + .../index-gateway/service-index-gateway.yaml | 36 + .../statefulset-index-gateway.yaml | 192 + .../templates/ingester/_helpers-ingester.tpl | 74 + .../loki/templates/ingester/hpa-zone-a.yaml | 55 + .../loki/templates/ingester/hpa-zone-b.yaml | 55 + .../loki/templates/ingester/hpa-zone-c.yaml | 55 + .../charts/loki/templates/ingester/hpa.yaml | 55 + .../poddisruptionbudget-ingester-rollout.yaml | 21 + .../poddisruptionbudget-ingester.yaml | 27 + .../ingester/service-ingester-headless.yaml | 35 + .../service-ingester-zone-a-headless.yaml | 38 + .../service-ingester-zone-b-headless.yaml | 38 + .../service-ingester-zone-c-headless.yaml | 38 + .../templates/ingester/service-ingester.yaml | 36 + .../ingester/statefulset-ingester-zone-a.yaml | 234 + .../ingester/statefulset-ingester-zone-b.yaml | 234 + .../ingester/statefulset-ingester-zone-c.yaml | 234 + .../ingester/statefulset-ingester.yaml | 205 + opencloud/charts/loki/templates/ingress.yaml | 40 + .../loki/templates/loki-canary/_helpers.tpl | 40 + .../loki/templates/loki-canary/daemonset.yaml | 123 + .../loki/templates/loki-canary/service.yaml | 34 + .../templates/loki-canary/serviceaccount.yaml | 21 + .../memcached/_memcached-statefulset.tpl | 178 + .../templates/memcached/_memcached-svc.tpl | 42 + .../monitoring/_helpers-monitoring.tpl | 47 + .../dashboards/_helpers-dashboards.tpl | 6 + .../monitoring/dashboards/configmap-1.yaml | 30 + .../monitoring/dashboards/configmap-2.yaml | 30 + .../templates/monitoring/grafana-agent.yaml | 100 + .../templates/monitoring/logs-instance.yaml | 30 + .../templates/monitoring/loki-alerts.yaml | 22 + .../loki/templates/monitoring/loki-rules.yaml | 23 + .../monitoring/metrics-instance.yaml | 30 + .../loki/templates/monitoring/pod-logs.yaml | 62 + .../templates/monitoring/servicemonitor.yaml | 63 + .../charts/loki/templates/networkpolicy.yaml | 203 + .../_helpers-pattern-ingester.tpl | 58 + .../statefulset-pattern-ingester.yaml | 187 + .../loki/templates/podsecuritypolicy.yaml | 41 + .../loki/templates/provisioner/_helpers.yaml | 32 + .../provisioner/job-provisioner.yaml | 147 + .../provisioner/role-provisioner.yaml | 21 + .../provisioner/rolebinding-provisioner.yaml | 26 + .../serviceaccount-provisioner.yaml | 18 + .../templates/querier/_helpers-querier.tpl | 32 + .../templates/querier/deployment-querier.yaml | 166 + .../charts/loki/templates/querier/hpa.yaml | 55 + .../querier/poddisruptionbudget-querier.yaml | 21 + .../templates/querier/service-querier.yaml | 36 + .../_helpers-query-frontend.tpl | 32 + .../deployment-query-frontend.yaml | 148 + .../loki/templates/query-frontend/hpa.yaml | 55 + .../poddisruptionbudget-query-frontend.yaml | 21 + .../service-query-frontend-headless.yaml | 46 + .../service-query-frontend.yaml | 44 + .../_helpers-query-scheduler.tpl | 40 + .../deployment-query-scheduler.yaml | 146 + .../poddisruptionbudget-query-scheduler.yaml | 21 + .../service-query-scheduler.yaml | 38 + .../loki/templates/read/_helpers-read.tpl | 32 + .../loki/templates/read/deployment-read.yaml | 163 + opencloud/charts/loki/templates/read/hpa.yaml | 55 + .../read/poddisruptionbudget-read.yaml | 15 + .../templates/read/service-read-headless.yaml | 41 + .../loki/templates/read/service-read.yaml | 37 + .../loki/templates/read/statefulset-read.yaml | 200 + .../poddisruptionbudget-results-cache.yaml | 16 + .../service-results-cache-headless.yaml | 1 + .../statefulset-results-cache.yaml | 1 + opencloud/charts/loki/templates/role.yaml | 36 + .../charts/loki/templates/rolebinding.yaml | 17 + .../loki/templates/ruler/_helpers-ruler.tpl | 47 + .../loki/templates/ruler/configmap-ruler.yaml | 15 + .../ruler/poddisruptionbudget-ruler.yaml | 21 + .../loki/templates/ruler/service-ruler.yaml | 37 + .../templates/ruler/statefulset-ruler.yaml | 184 + .../loki/templates/runtime-configmap.yaml | 10 + .../charts/loki/templates/secret-license.yaml | 11 + .../templates/securitycontextconstraints.yaml | 40 + .../loki/templates/service-memberlist.yaml | 29 + .../charts/loki/templates/serviceaccount.yaml | 21 + .../single-binary/_helpers-single-binary.tpl | 34 + .../loki/templates/single-binary/hpa.yaml | 51 + .../loki/templates/single-binary/pdb.yaml | 16 + .../single-binary/service-headless.yaml | 35 + .../loki/templates/single-binary/service.yaml | 40 + .../templates/single-binary/statefulset.yaml | 278 + .../table-manager/_helpers-table-manager.tpl | 32 + .../deployment-table-manager.yaml | 126 + .../table-manager/service-table-manager.yaml | 36 + .../servicemonitor-table-manager.yaml | 57 + .../charts/loki/templates/tests/_helpers.tpl | 16 + .../loki/templates/tests/test-canary.yaml | 36 + .../loki/templates/tokengen/_helpers.yaml | 22 + .../tokengen/clusterrole-tokengen.yaml | 21 + .../tokengen/clusterrolebinding-tokengen.yaml | 25 + .../loki/templates/tokengen/job-tokengen.yaml | 143 + .../tokengen/serviceaccount-tokengen.yaml | 18 + opencloud/charts/loki/templates/validate.yaml | 41 + .../loki/templates/write/_helpers-write.tpl | 32 + .../charts/loki/templates/write/hpa.yaml | 51 + .../write/poddisruptionbudget-write.yaml | 15 + .../write/service-write-headless.yaml | 41 + .../loki/templates/write/service-write.yaml | 37 + .../templates/write/statefulset-write.yaml | 217 + opencloud/charts/loki/test/config_test.go | 220 + opencloud/charts/loki/values.yaml | 3618 ++++++++++ opencloud/templates/oc-auth/deployment.yaml | 2 +- 318 files changed, 47355 insertions(+), 1 deletion(-) create mode 100644 opencloud/charts/grafana/.helmignore create mode 100644 opencloud/charts/grafana/Chart.yaml create mode 100644 opencloud/charts/grafana/README.md create mode 100644 opencloud/charts/grafana/ci/default-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-affinity-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-dashboard-json-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-dashboard-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-extraconfigmapmounts-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-image-renderer-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-nondefault-values.yaml create mode 100644 opencloud/charts/grafana/ci/with-persistence.yaml create mode 100644 opencloud/charts/grafana/ci/with-sidecars-envvaluefrom-values.yaml create mode 100644 opencloud/charts/grafana/dashboards/custom-dashboard.json create mode 100644 opencloud/charts/grafana/templates/NOTES.txt create mode 100644 opencloud/charts/grafana/templates/_config.tpl create mode 100644 opencloud/charts/grafana/templates/_helpers.tpl create mode 100644 opencloud/charts/grafana/templates/_pod.tpl create mode 100644 opencloud/charts/grafana/templates/clusterrole.yaml create mode 100644 opencloud/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 opencloud/charts/grafana/templates/configSecret.yaml create mode 100644 opencloud/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 opencloud/charts/grafana/templates/configmap.yaml create mode 100644 opencloud/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 opencloud/charts/grafana/templates/deployment.yaml create mode 100644 opencloud/charts/grafana/templates/extra-manifests.yaml create mode 100644 opencloud/charts/grafana/templates/headless-service.yaml create mode 100644 opencloud/charts/grafana/templates/hpa.yaml create mode 100644 opencloud/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 opencloud/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 opencloud/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 opencloud/charts/grafana/templates/image-renderer-service.yaml create mode 100644 opencloud/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 opencloud/charts/grafana/templates/ingress.yaml create mode 100644 opencloud/charts/grafana/templates/networkpolicy.yaml create mode 100644 opencloud/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 opencloud/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 opencloud/charts/grafana/templates/pvc.yaml create mode 100644 opencloud/charts/grafana/templates/role.yaml create mode 100644 opencloud/charts/grafana/templates/rolebinding.yaml create mode 100644 opencloud/charts/grafana/templates/route.yaml create mode 100644 opencloud/charts/grafana/templates/secret-env.yaml create mode 100644 opencloud/charts/grafana/templates/secret.yaml create mode 100644 opencloud/charts/grafana/templates/service.yaml create mode 100644 opencloud/charts/grafana/templates/serviceaccount.yaml create mode 100644 opencloud/charts/grafana/templates/servicemonitor.yaml create mode 100644 opencloud/charts/grafana/templates/statefulset.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test-role.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 opencloud/charts/grafana/templates/tests/test.yaml create mode 100644 opencloud/charts/grafana/values.yaml create mode 100644 opencloud/charts/loki/.helmignore create mode 100644 opencloud/charts/loki/Chart.lock create mode 100644 opencloud/charts/loki/Chart.yaml create mode 100644 opencloud/charts/loki/Makefile create mode 100644 opencloud/charts/loki/README.md create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/.helmignore create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/Chart.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/README.md create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/README.md.gotmpl create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_podmonitors.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_probes.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_servicemonitors.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_grafanaagents.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_integrations.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_logsinstances.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_metricsinstances.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_podlogs.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/_helpers.tpl create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrole.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrolebinding.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-deployment.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-serviceaccount.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/templates/tests/test-grafanaagent.yaml create mode 100644 opencloud/charts/loki/charts/grafana-agent-operator/values.yaml create mode 100644 opencloud/charts/loki/charts/minio/.helmignore create mode 100644 opencloud/charts/loki/charts/minio/Chart.yaml create mode 100644 opencloud/charts/loki/charts/minio/README.md create mode 100644 opencloud/charts/loki/charts/minio/templates/NOTES.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_create_bucket.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_create_policy.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_create_svcacct.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_create_user.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_custom_command.txt create mode 100644 opencloud/charts/loki/charts/minio/templates/_helper_policy.tpl create mode 100644 opencloud/charts/loki/charts/minio/templates/_helpers.tpl create mode 100644 opencloud/charts/loki/charts/minio/templates/ciliumnetworkpolicy.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/configmap.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/console-ingress.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/console-service.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/deployment.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/ingress.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/networkpolicy.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/poddisruptionbudget.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/post-job.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/pvc.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/secrets.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/securitycontextconstraints.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/service.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/serviceaccount.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/servicemonitor.yaml create mode 100644 opencloud/charts/loki/charts/minio/templates/statefulset.yaml create mode 100644 opencloud/charts/loki/charts/minio/values.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/.helmignore create mode 100644 opencloud/charts/loki/charts/rollout-operator/Chart.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/README.md create mode 100644 opencloud/charts/loki/charts/rollout-operator/README.md.gotmpl create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/NOTES.txt create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/_helpers.tpl create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/deployment.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/role.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/rolebinding.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/service.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/serviceaccount.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/templates/servicemonitor.yaml create mode 100644 opencloud/charts/loki/charts/rollout-operator/values.yaml create mode 100644 opencloud/charts/loki/distributed-values.yaml create mode 100644 opencloud/charts/loki/docs/examples/README.md create mode 100644 opencloud/charts/loki/docs/examples/enterprise/README.md create mode 100644 opencloud/charts/loki/docs/examples/enterprise/enterprise-secrets.yaml create mode 100644 opencloud/charts/loki/docs/examples/enterprise/overrides-enterprise-gcs.yaml create mode 100644 opencloud/charts/loki/docs/examples/oss/README.md create mode 100644 opencloud/charts/loki/docs/examples/oss/oss-secrets.yaml create mode 100644 opencloud/charts/loki/docs/examples/oss/overrides-oss-gcs.yaml create mode 100644 opencloud/charts/loki/reference.md.gotmpl create mode 100644 opencloud/charts/loki/scenarios/README.md create mode 100644 opencloud/charts/loki/scenarios/default-single-binary-values.yaml create mode 100644 opencloud/charts/loki/scenarios/default-values.yaml create mode 100644 opencloud/charts/loki/scenarios/images/added.png create mode 100644 opencloud/charts/loki/scenarios/images/img.png create mode 100644 opencloud/charts/loki/scenarios/images/modified.png create mode 100644 opencloud/charts/loki/scenarios/images/removed.png create mode 100644 opencloud/charts/loki/scenarios/ingress-values.yaml create mode 100644 opencloud/charts/loki/scenarios/legacy-monitoring-values.yaml create mode 100644 opencloud/charts/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml create mode 100644 opencloud/charts/loki/simple-scalable-values.yaml create mode 100644 opencloud/charts/loki/single-binary-values.yaml create mode 100644 opencloud/charts/loki/src/.yamllint.yaml create mode 100644 opencloud/charts/loki/src/alerts.yaml.tpl create mode 100644 opencloud/charts/loki/src/dashboards/loki-chunks.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-deletion.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-logs.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-mixin-recording-rules.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-operational.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-reads-resources.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-reads.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-retention.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-writes-resources.json create mode 100644 opencloud/charts/loki/src/dashboards/loki-writes.json create mode 100644 opencloud/charts/loki/src/helm-test/Dockerfile create mode 100644 opencloud/charts/loki/src/helm-test/README.md create mode 100644 opencloud/charts/loki/src/helm-test/canary_test.go create mode 100644 opencloud/charts/loki/src/helm-test/default.nix create mode 100644 opencloud/charts/loki/src/rules.yaml.tpl create mode 100644 opencloud/charts/loki/templates/NOTES.txt create mode 100644 opencloud/charts/loki/templates/_helpers.tpl create mode 100644 opencloud/charts/loki/templates/admin-api/_helpers.yaml create mode 100644 opencloud/charts/loki/templates/admin-api/deployment-admin-api.yaml create mode 100644 opencloud/charts/loki/templates/admin-api/service-admin-api.yaml create mode 100644 opencloud/charts/loki/templates/backend/_helpers-backend.tpl create mode 100644 opencloud/charts/loki/templates/backend/clusterrole.yaml create mode 100644 opencloud/charts/loki/templates/backend/clusterrolebinding.yaml create mode 100644 opencloud/charts/loki/templates/backend/hpa.yaml create mode 100644 opencloud/charts/loki/templates/backend/poddisruptionbudget-backend.yaml create mode 100644 opencloud/charts/loki/templates/backend/query-scheduler-discovery.yaml create mode 100644 opencloud/charts/loki/templates/backend/service-backend-headless.yaml create mode 100644 opencloud/charts/loki/templates/backend/service-backend.yaml create mode 100644 opencloud/charts/loki/templates/backend/statefulset-backend.yaml create mode 100644 opencloud/charts/loki/templates/bloom-builder/_helpers-bloom-builder.tpl create mode 100644 opencloud/charts/loki/templates/bloom-builder/deployment-bloom-builder.yaml create mode 100644 opencloud/charts/loki/templates/bloom-builder/hpa.yaml create mode 100644 opencloud/charts/loki/templates/bloom-builder/poddisruptionbudget-bloom-builder.yaml create mode 100644 opencloud/charts/loki/templates/bloom-builder/service-bloom-builder-headless.yaml create mode 100644 opencloud/charts/loki/templates/bloom-builder/service-bloom-builder.yaml create mode 100644 opencloud/charts/loki/templates/bloom-gateway/_helpers-bloom-gateway.tpl create mode 100644 opencloud/charts/loki/templates/bloom-gateway/service-bloom-gateway-headless.yaml create mode 100644 opencloud/charts/loki/templates/bloom-gateway/statefulset-bloom-gateway.yaml create mode 100644 opencloud/charts/loki/templates/bloom-planner/_helpers-bloom-planner.tpl create mode 100644 opencloud/charts/loki/templates/bloom-planner/service-bloom-planner-headless.yaml create mode 100644 opencloud/charts/loki/templates/bloom-planner/statefulset-bloom-planner.yaml create mode 100644 opencloud/charts/loki/templates/chunks-cache/poddisruptionbudget-chunks-cache.yaml create mode 100644 opencloud/charts/loki/templates/chunks-cache/service-chunks-cache-headless.yaml create mode 100644 opencloud/charts/loki/templates/chunks-cache/statefulset-chunks-cache.yaml create mode 100644 opencloud/charts/loki/templates/ciliumnetworkpolicy.yaml create mode 100644 opencloud/charts/loki/templates/compactor/_helpers-compactor.tpl create mode 100644 opencloud/charts/loki/templates/compactor/service-compactor.yaml create mode 100644 opencloud/charts/loki/templates/compactor/statefulset-compactor.yaml create mode 100644 opencloud/charts/loki/templates/config.yaml create mode 100644 opencloud/charts/loki/templates/distributor/_helpers-distributor.tpl create mode 100644 opencloud/charts/loki/templates/distributor/deployment-distributor.yaml create mode 100644 opencloud/charts/loki/templates/distributor/hpa.yaml create mode 100644 opencloud/charts/loki/templates/distributor/poddisruptionbudget-distributor.yaml create mode 100644 opencloud/charts/loki/templates/distributor/service-distributor-headless.yaml create mode 100644 opencloud/charts/loki/templates/distributor/service-distributor.yaml create mode 100644 opencloud/charts/loki/templates/extra-manifests.yaml create mode 100644 opencloud/charts/loki/templates/gateway/_helpers-gateway.tpl create mode 100644 opencloud/charts/loki/templates/gateway/configmap-gateway.yaml create mode 100644 opencloud/charts/loki/templates/gateway/deployment-gateway-enterprise.yaml create mode 100644 opencloud/charts/loki/templates/gateway/deployment-gateway-nginx.yaml create mode 100644 opencloud/charts/loki/templates/gateway/hpa.yaml create mode 100644 opencloud/charts/loki/templates/gateway/ingress-gateway.yaml create mode 100644 opencloud/charts/loki/templates/gateway/poddisruptionbudget-gateway.yaml create mode 100644 opencloud/charts/loki/templates/gateway/secret-gateway.yaml create mode 100644 opencloud/charts/loki/templates/gateway/service-gateway.yaml create mode 100644 opencloud/charts/loki/templates/index-gateway/_helpers-index-gateway.tpl create mode 100644 opencloud/charts/loki/templates/index-gateway/poddisruptionbudget-index-gateway.yaml create mode 100644 opencloud/charts/loki/templates/index-gateway/service-index-gateway-headless.yaml create mode 100644 opencloud/charts/loki/templates/index-gateway/service-index-gateway.yaml create mode 100644 opencloud/charts/loki/templates/index-gateway/statefulset-index-gateway.yaml create mode 100644 opencloud/charts/loki/templates/ingester/_helpers-ingester.tpl create mode 100644 opencloud/charts/loki/templates/ingester/hpa-zone-a.yaml create mode 100644 opencloud/charts/loki/templates/ingester/hpa-zone-b.yaml create mode 100644 opencloud/charts/loki/templates/ingester/hpa-zone-c.yaml create mode 100644 opencloud/charts/loki/templates/ingester/hpa.yaml create mode 100644 opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml create mode 100644 opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester.yaml create mode 100644 opencloud/charts/loki/templates/ingester/service-ingester-headless.yaml create mode 100644 opencloud/charts/loki/templates/ingester/service-ingester-zone-a-headless.yaml create mode 100644 opencloud/charts/loki/templates/ingester/service-ingester-zone-b-headless.yaml create mode 100644 opencloud/charts/loki/templates/ingester/service-ingester-zone-c-headless.yaml create mode 100644 opencloud/charts/loki/templates/ingester/service-ingester.yaml create mode 100644 opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-a.yaml create mode 100644 opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-b.yaml create mode 100644 opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-c.yaml create mode 100644 opencloud/charts/loki/templates/ingester/statefulset-ingester.yaml create mode 100644 opencloud/charts/loki/templates/ingress.yaml create mode 100644 opencloud/charts/loki/templates/loki-canary/_helpers.tpl create mode 100644 opencloud/charts/loki/templates/loki-canary/daemonset.yaml create mode 100644 opencloud/charts/loki/templates/loki-canary/service.yaml create mode 100644 opencloud/charts/loki/templates/loki-canary/serviceaccount.yaml create mode 100644 opencloud/charts/loki/templates/memcached/_memcached-statefulset.tpl create mode 100644 opencloud/charts/loki/templates/memcached/_memcached-svc.tpl create mode 100644 opencloud/charts/loki/templates/monitoring/_helpers-monitoring.tpl create mode 100644 opencloud/charts/loki/templates/monitoring/dashboards/_helpers-dashboards.tpl create mode 100644 opencloud/charts/loki/templates/monitoring/dashboards/configmap-1.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/dashboards/configmap-2.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/grafana-agent.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/logs-instance.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/loki-alerts.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/loki-rules.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/metrics-instance.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/pod-logs.yaml create mode 100644 opencloud/charts/loki/templates/monitoring/servicemonitor.yaml create mode 100644 opencloud/charts/loki/templates/networkpolicy.yaml create mode 100644 opencloud/charts/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl create mode 100644 opencloud/charts/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml create mode 100644 opencloud/charts/loki/templates/podsecuritypolicy.yaml create mode 100644 opencloud/charts/loki/templates/provisioner/_helpers.yaml create mode 100644 opencloud/charts/loki/templates/provisioner/job-provisioner.yaml create mode 100644 opencloud/charts/loki/templates/provisioner/role-provisioner.yaml create mode 100644 opencloud/charts/loki/templates/provisioner/rolebinding-provisioner.yaml create mode 100644 opencloud/charts/loki/templates/provisioner/serviceaccount-provisioner.yaml create mode 100644 opencloud/charts/loki/templates/querier/_helpers-querier.tpl create mode 100644 opencloud/charts/loki/templates/querier/deployment-querier.yaml create mode 100644 opencloud/charts/loki/templates/querier/hpa.yaml create mode 100644 opencloud/charts/loki/templates/querier/poddisruptionbudget-querier.yaml create mode 100644 opencloud/charts/loki/templates/querier/service-querier.yaml create mode 100644 opencloud/charts/loki/templates/query-frontend/_helpers-query-frontend.tpl create mode 100644 opencloud/charts/loki/templates/query-frontend/deployment-query-frontend.yaml create mode 100644 opencloud/charts/loki/templates/query-frontend/hpa.yaml create mode 100644 opencloud/charts/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml create mode 100644 opencloud/charts/loki/templates/query-frontend/service-query-frontend-headless.yaml create mode 100644 opencloud/charts/loki/templates/query-frontend/service-query-frontend.yaml create mode 100644 opencloud/charts/loki/templates/query-scheduler/_helpers-query-scheduler.tpl create mode 100644 opencloud/charts/loki/templates/query-scheduler/deployment-query-scheduler.yaml create mode 100644 opencloud/charts/loki/templates/query-scheduler/poddisruptionbudget-query-scheduler.yaml create mode 100644 opencloud/charts/loki/templates/query-scheduler/service-query-scheduler.yaml create mode 100644 opencloud/charts/loki/templates/read/_helpers-read.tpl create mode 100644 opencloud/charts/loki/templates/read/deployment-read.yaml create mode 100644 opencloud/charts/loki/templates/read/hpa.yaml create mode 100644 opencloud/charts/loki/templates/read/poddisruptionbudget-read.yaml create mode 100644 opencloud/charts/loki/templates/read/service-read-headless.yaml create mode 100644 opencloud/charts/loki/templates/read/service-read.yaml create mode 100644 opencloud/charts/loki/templates/read/statefulset-read.yaml create mode 100644 opencloud/charts/loki/templates/results-cache/poddisruptionbudget-results-cache.yaml create mode 100644 opencloud/charts/loki/templates/results-cache/service-results-cache-headless.yaml create mode 100644 opencloud/charts/loki/templates/results-cache/statefulset-results-cache.yaml create mode 100644 opencloud/charts/loki/templates/role.yaml create mode 100644 opencloud/charts/loki/templates/rolebinding.yaml create mode 100644 opencloud/charts/loki/templates/ruler/_helpers-ruler.tpl create mode 100644 opencloud/charts/loki/templates/ruler/configmap-ruler.yaml create mode 100644 opencloud/charts/loki/templates/ruler/poddisruptionbudget-ruler.yaml create mode 100644 opencloud/charts/loki/templates/ruler/service-ruler.yaml create mode 100644 opencloud/charts/loki/templates/ruler/statefulset-ruler.yaml create mode 100644 opencloud/charts/loki/templates/runtime-configmap.yaml create mode 100644 opencloud/charts/loki/templates/secret-license.yaml create mode 100644 opencloud/charts/loki/templates/securitycontextconstraints.yaml create mode 100644 opencloud/charts/loki/templates/service-memberlist.yaml create mode 100644 opencloud/charts/loki/templates/serviceaccount.yaml create mode 100644 opencloud/charts/loki/templates/single-binary/_helpers-single-binary.tpl create mode 100644 opencloud/charts/loki/templates/single-binary/hpa.yaml create mode 100644 opencloud/charts/loki/templates/single-binary/pdb.yaml create mode 100644 opencloud/charts/loki/templates/single-binary/service-headless.yaml create mode 100644 opencloud/charts/loki/templates/single-binary/service.yaml create mode 100644 opencloud/charts/loki/templates/single-binary/statefulset.yaml create mode 100644 opencloud/charts/loki/templates/table-manager/_helpers-table-manager.tpl create mode 100644 opencloud/charts/loki/templates/table-manager/deployment-table-manager.yaml create mode 100644 opencloud/charts/loki/templates/table-manager/service-table-manager.yaml create mode 100644 opencloud/charts/loki/templates/table-manager/servicemonitor-table-manager.yaml create mode 100644 opencloud/charts/loki/templates/tests/_helpers.tpl create mode 100644 opencloud/charts/loki/templates/tests/test-canary.yaml create mode 100644 opencloud/charts/loki/templates/tokengen/_helpers.yaml create mode 100644 opencloud/charts/loki/templates/tokengen/clusterrole-tokengen.yaml create mode 100644 opencloud/charts/loki/templates/tokengen/clusterrolebinding-tokengen.yaml create mode 100644 opencloud/charts/loki/templates/tokengen/job-tokengen.yaml create mode 100644 opencloud/charts/loki/templates/tokengen/serviceaccount-tokengen.yaml create mode 100644 opencloud/charts/loki/templates/validate.yaml create mode 100644 opencloud/charts/loki/templates/write/_helpers-write.tpl create mode 100644 opencloud/charts/loki/templates/write/hpa.yaml create mode 100644 opencloud/charts/loki/templates/write/poddisruptionbudget-write.yaml create mode 100644 opencloud/charts/loki/templates/write/service-write-headless.yaml create mode 100644 opencloud/charts/loki/templates/write/service-write.yaml create mode 100644 opencloud/charts/loki/templates/write/statefulset-write.yaml create mode 100644 opencloud/charts/loki/test/config_test.go create mode 100644 opencloud/charts/loki/values.yaml diff --git a/opencloud/charts/grafana/.helmignore b/opencloud/charts/grafana/.helmignore new file mode 100644 index 0000000..8cade13 --- /dev/null +++ b/opencloud/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/opencloud/charts/grafana/Chart.yaml b/opencloud/charts/grafana/Chart.yaml new file mode 100644 index 0000000..76628eb --- /dev/null +++ b/opencloud/charts/grafana/Chart.yaml @@ -0,0 +1,35 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana +apiVersion: v2 +appVersion: 11.3.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: ^1.8.0-0 +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +- email: github@jkroepke.de + name: jkroepke +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 8.6.4 diff --git a/opencloud/charts/grafana/README.md b/opencloud/charts/grafana/README.md new file mode 100644 index 0000000..a78b123 --- /dev/null +++ b/opencloud/charts/grafana/README.md @@ -0,0 +1,783 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.ipFamilies` | Kubernetes service IP families | `[]` | +| `service.ipFamilyPolicy` | Kubernetes service IP family policy | `""` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `persistence.disableWarning` | Hide NOTES warning, useful when persisting to a database | `false` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.28.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.folderUid` | Allows you to specify the static UID for the logical folder above | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.extraConfigmapMounts` | Additional image-renderer configMap volume mounts (values are templated) | `[]` | +| `imageRenderer.extraSecretMounts` | Additional image-renderer secret volume mounts | `[]` | +| `imageRenderer.extraVolumeMounts` | Additional image-renderer volume mounts | `[]` | +| `imageRenderer.extraVolumes` | Additional image-renderer volumes | `[]` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.serverURL` | Remote image renderer url | `''` | +| `imageRenderer.renderingCallbackURL` | Callback url for the Grafana image renderer | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources + +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/opencloud/charts/grafana/ci/default-values.yaml b/opencloud/charts/grafana/ci/default-values.yaml new file mode 100644 index 0000000..fc2ba60 --- /dev/null +++ b/opencloud/charts/grafana/ci/default-values.yaml @@ -0,0 +1 @@ +# Leave this file empty to ensure that CI runs builds against the default configuration in values.yaml. diff --git a/opencloud/charts/grafana/ci/with-affinity-values.yaml b/opencloud/charts/grafana/ci/with-affinity-values.yaml new file mode 100644 index 0000000..f5b9b53 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-affinity-values.yaml @@ -0,0 +1,16 @@ +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: grafana-test + app.kubernetes.io/name: grafana + topologyKey: failure-domain.beta.kubernetes.io/zone + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: grafana-test + app.kubernetes.io/name: grafana + topologyKey: kubernetes.io/hostname diff --git a/opencloud/charts/grafana/ci/with-dashboard-json-values.yaml b/opencloud/charts/grafana/ci/with-dashboard-json-values.yaml new file mode 100644 index 0000000..e0c4e41 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-dashboard-json-values.yaml @@ -0,0 +1,53 @@ +dashboards: + my-provider: + my-awesome-dashboard: + # An empty but valid dashboard + json: | + { + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.3.5" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [], + "schemaVersion": 19, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s"] + }, + "timezone": "", + "title": "Dummy Dashboard", + "uid": "IdcYQooWk", + "version": 1 + } + datasource: Prometheus diff --git a/opencloud/charts/grafana/ci/with-dashboard-values.yaml b/opencloud/charts/grafana/ci/with-dashboard-values.yaml new file mode 100644 index 0000000..7b662c5 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-dashboard-values.yaml @@ -0,0 +1,19 @@ +dashboards: + my-provider: + my-awesome-dashboard: + gnetId: 10000 + revision: 1 + datasource: Prometheus +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'my-provider' + orgId: 1 + folder: '' + type: file + updateIntervalSeconds: 10 + disableDeletion: true + editable: true + options: + path: /var/lib/grafana/dashboards/my-provider diff --git a/opencloud/charts/grafana/ci/with-extraconfigmapmounts-values.yaml b/opencloud/charts/grafana/ci/with-extraconfigmapmounts-values.yaml new file mode 100644 index 0000000..5cc44a0 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-extraconfigmapmounts-values.yaml @@ -0,0 +1,7 @@ +extraConfigmapMounts: + - name: '{{ include "grafana.fullname" . }}' + configMap: '{{ include "grafana.fullname" . }}' + mountPath: /var/lib/grafana/dashboards/test-dashboard.json + # This is not a realistic test, but for this we only care about extraConfigmapMounts not being empty and pointing to an existing ConfigMap + subPath: grafana.ini + readOnly: true diff --git a/opencloud/charts/grafana/ci/with-image-renderer-values.yaml b/opencloud/charts/grafana/ci/with-image-renderer-values.yaml new file mode 100644 index 0000000..06c0bda --- /dev/null +++ b/opencloud/charts/grafana/ci/with-image-renderer-values.yaml @@ -0,0 +1,107 @@ +podLabels: + customLableA: Aaaaa +imageRenderer: + enabled: true + env: + RENDERING_ARGS: --disable-gpu,--window-size=1280x758 + RENDERING_MODE: clustered + podLabels: + customLableB: Bbbbb + networkPolicy: + limitIngress: true + limitEgress: true + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 500m + memory: 50Mi + extraVolumes: + - name: empty-renderer-volume + emtpyDir: {} + extraVolumeMounts: + - mountPath: /tmp/renderer + name: empty-renderer-volume + extraConfigmapMounts: + - name: renderer-config + mountPath: /usr/src/app/config.json + subPath: renderer-config.json + configMap: image-renderer-config + extraSecretMounts: + - name: renderer-certificate + mountPath: /usr/src/app/certs/ + secretName: image-renderer-certificate + readOnly: true + +extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: image-renderer-config + data: + renderer-config.json: | + { + "service": { + "host": null, + "port": 8081, + "protocol": "http", + "certFile": "", + "certKey": "", + + "metrics": { + "enabled": true, + "collectDefaultMetrics": true, + "requestDurationBuckets": [1, 5, 7, 9, 11, 13, 15, 20, 30] + }, + + "logging": { + "level": "info", + "console": { + "json": true, + "colorize": false + } + }, + + "security": { + "authToken": "-" + } + }, + "rendering": { + "chromeBin": null, + "args": ["--no-sandbox", "--disable-gpu"], + "ignoresHttpsErrors": false, + + "timezone": null, + "acceptLanguage": null, + "width": 1000, + "height": 500, + "deviceScaleFactor": 1, + "maxWidth": 3080, + "maxHeight": 3000, + "maxDeviceScaleFactor": 4, + "pageZoomLevel": 1, + "headed": false, + + "mode": "default", + "emulateNetworkConditions": false, + "clustering": { + "monitor": false, + "mode": "browser", + "maxConcurrency": 5, + "timeout": 30 + }, + + "verboseLogging": false, + "dumpio": false, + "timingMetrics": false + } + } + - apiVersion: v1 + kind: Secret + metadata: + name: image-renderer-certificate + type: Opaque + data: + # Decodes to 'PLACEHOLDER CERTIFICATE' + not-a-real-certificate: UExBQ0VIT0xERVIgQ0VSVElGSUNBVEU= diff --git a/opencloud/charts/grafana/ci/with-nondefault-values.yaml b/opencloud/charts/grafana/ci/with-nondefault-values.yaml new file mode 100644 index 0000000..4848489 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-nondefault-values.yaml @@ -0,0 +1,32 @@ +global: + environment: prod +ingress: + enabled: true + hosts: + - monitoring-{{ .Values.global.environment }}.example.com + +route: + main: + enabled: true + labels: + app: monitoring-prometheus + hostnames: + - "*.example.com" + - "{{ .Values.global.environment }}.example.com" + filters: + - type: RequestHeaderModifier + requestHeaderModifier: + set: + - name: my-header-name + value: my-new-header-value + additionalRules: + - filters: + - type: RequestHeaderModifier + requestHeaderModifier: + set: + - name: my-header-name + value: my-new-header-value + matches: + - path: + type: PathPrefix + value: /foo/ diff --git a/opencloud/charts/grafana/ci/with-persistence.yaml b/opencloud/charts/grafana/ci/with-persistence.yaml new file mode 100644 index 0000000..b92ca02 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-persistence.yaml @@ -0,0 +1,3 @@ +persistence: + type: pvc + enabled: true diff --git a/opencloud/charts/grafana/ci/with-sidecars-envvaluefrom-values.yaml b/opencloud/charts/grafana/ci/with-sidecars-envvaluefrom-values.yaml new file mode 100644 index 0000000..a6935e5 --- /dev/null +++ b/opencloud/charts/grafana/ci/with-sidecars-envvaluefrom-values.yaml @@ -0,0 +1,38 @@ +extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ include "grafana.fullname" . }}-test' + data: + var1: "value1" + - apiVersion: v1 + kind: Secret + metadata: + name: '{{ include "grafana.fullname" . }}-test' + type: Opaque + data: + var2: "dmFsdWUy" + +sidecar: + dashboards: + enabled: true + envValueFrom: + VAR1: + configMapKeyRef: + name: '{{ include "grafana.fullname" . }}-test' + key: var1 + VAR2: + secretKeyRef: + name: '{{ include "grafana.fullname" . }}-test' + key: var2 + datasources: + enabled: true + envValueFrom: + VAR1: + configMapKeyRef: + name: '{{ include "grafana.fullname" . }}-test' + key: var1 + VAR2: + secretKeyRef: + name: '{{ include "grafana.fullname" . }}-test' + key: var2 diff --git a/opencloud/charts/grafana/dashboards/custom-dashboard.json b/opencloud/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/opencloud/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/opencloud/charts/grafana/templates/NOTES.txt b/opencloud/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000..a40f666 --- /dev/null +++ b/opencloud/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if and (not .Values.persistence.enabled) (not .Values.persistence.disableWarning) }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/opencloud/charts/grafana/templates/_config.tpl b/opencloud/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000..8897620 --- /dev/null +++ b/opencloud/charts/grafana/templates/_config.tpl @@ -0,0 +1,176 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "slice" $elemVal }} + {{ $elem }} = {{ toJson $elemVal }} + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "slice" $elemVal }} + {{ $elem }} = {{ toJson $elemVal }} + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + folderUid: '{{ .Values.sidecar.dashboards.provider.folderUid }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/opencloud/charts/grafana/templates/_helpers.tpl b/opencloud/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000..be9c1c5 --- /dev/null +++ b/opencloud/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,274 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | trunc 63 | trimSuffix "-" | quote }} +{{- end }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | trunc 63 | trimSuffix "-" | quote }} +{{- end }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/opencloud/charts/grafana/templates/_pod.tpl b/opencloud/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000..bf55e43 --- /dev/null +++ b/opencloud/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1389 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := .Values.global.imageRegistry | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}/{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := .Values.global.imageRegistry | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}/{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.maxTotalRetries }} + - name: REQ_RETRY_TOTAL + value: "{{ .Values.sidecar.alerts.maxTotalRetries }}" + {{- end }} + {{- if .Values.sidecar.alerts.maxConnectRetries }} + - name: REQ_RETRY_CONNECT + value: "{{ .Values.sidecar.alerts.maxConnectRetries }}" + {{- end }} + {{- if .Values.sidecar.alerts.maxReadRetries }} + - name: REQ_RETRY_READ + value: "{{ .Values.sidecar.alerts.maxReadRetries }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.dashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- if .Values.sidecar.dashboards.maxTotalRetries }} + - name: REQ_RETRY_TOTAL + value: "{{ .Values.sidecar.dashboards.maxTotalRetries }}" + {{- end }} + {{- if .Values.sidecar.dashboards.maxConnectRetries }} + - name: REQ_RETRY_CONNECT + value: "{{ .Values.sidecar.dashboards.maxConnectRetries }}" + {{- end }} + {{- if .Values.sidecar.dashboards.maxReadRetries }} + - name: REQ_RETRY_READ + value: "{{ .Values.sidecar.dashboards.maxReadRetries }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.maxTotalRetries }} + - name: REQ_RETRY_TOTAL + value: "{{ .Values.sidecar.datasources.maxTotalRetries }}" + {{- end }} + {{- if .Values.sidecar.datasources.maxConnectRetries }} + - name: REQ_RETRY_CONNECT + value: "{{ .Values.sidecar.datasources.maxConnectRetries }}" + {{- end }} + {{- if .Values.sidecar.datasources.maxReadRetries }} + - name: REQ_RETRY_READ + value: "{{ .Values.sidecar.datasources.maxReadRetries }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- with .Values.sidecar.datasources.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.maxTotalRetries }} + - name: REQ_RETRY_TOTAL + value: "{{ .Values.sidecar.notifiers.maxTotalRetries }}" + {{- end }} + {{- if .Values.sidecar.notifiers.maxConnectRetries }} + - name: REQ_RETRY_CONNECT + value: "{{ .Values.sidecar.notifiers.maxConnectRetries }}" + {{- end }} + {{- if .Values.sidecar.notifiers.maxReadRetries }} + - name: REQ_RETRY_READ + value: "{{ .Values.sidecar.notifiers.maxReadRetries }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- with .Values.sidecar.notifiers.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := .Values.global.imageRegistry | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.maxTotalRetries }} + - name: REQ_RETRY_TOTAL + value: "{{ .Values.sidecar.plugins.maxTotalRetries }}" + {{- end }} + {{- if .Values.sidecar.plugins.maxConnectRetries }} + - name: REQ_RETRY_CONNECT + value: "{{ .Values.sidecar.plugins.maxConnectRetries }}" + {{- end }} + {{- if .Values.sidecar.plugins.maxReadRetries }} + - name: REQ_RETRY_READ + value: "{{ .Values.sidecar.plugins.maxReadRetries }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- with .Values.sidecar.plugins.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := .Values.global.imageRegistry | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + {{- if .Values.imageRenderer.serverURL }} + value: {{ .Values.imageRenderer.serverURL | quote }} + {{- else }} + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + {{- end }} + - name: GF_RENDERING_CALLBACK_URL + {{- if .Values.imageRenderer.renderingCallbackURL }} + value: {{ .Values.imageRenderer.renderingCallbackURL | quote }} + {{- else }} + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- with .Values.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .optional }} + optional: {{ . }} + {{- end }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .optional }} + optional: {{ . }} + {{- end }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/clusterrole.yaml b/opencloud/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000..3af4b62 --- /dev/null +++ b/opencloud/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/opencloud/charts/grafana/templates/clusterrolebinding.yaml b/opencloud/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..bda9431 --- /dev/null +++ b/opencloud/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/opencloud/charts/grafana/templates/configSecret.yaml b/opencloud/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000..55574b9 --- /dev/null +++ b/opencloud/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/configmap-dashboard-provider.yaml b/opencloud/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000..b412c4d --- /dev/null +++ b/opencloud/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/configmap.yaml b/opencloud/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000..0a2edf4 --- /dev/null +++ b/opencloud/charts/grafana/templates/configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- if or .Values.configMapAnnotations .Values.annotations }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.configMapAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/dashboards-json-configmap.yaml b/opencloud/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000..df0ed0d --- /dev/null +++ b/opencloud/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,35 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/opencloud/charts/grafana/templates/deployment.yaml b/opencloud/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000..7bf7521 --- /dev/null +++ b/opencloud/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if (not .Values.autoscaling.enabled) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/extra-manifests.yaml b/opencloud/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000..a9bb3b6 --- /dev/null +++ b/opencloud/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/opencloud/charts/grafana/templates/headless-service.yaml b/opencloud/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000..3028589 --- /dev/null +++ b/opencloud/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/opencloud/charts/grafana/templates/hpa.yaml b/opencloud/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000..d9624b2 --- /dev/null +++ b/opencloud/charts/grafana/templates/hpa.yaml @@ -0,0 +1,51 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/image-renderer-deployment.yaml b/opencloud/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000..4828aa3 --- /dev/null +++ b/opencloud/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,199 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := .Values.global.imageRegistry | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}/{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- range .Values.imageRenderer.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.imageRenderer.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.imageRenderer.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} + {{- range .Values.imageRenderer.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- end }} + {{- range .Values.imageRenderer.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 12 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- end }} + {{- end }} + {{- range .Values.imageRenderer.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/image-renderer-hpa.yaml b/opencloud/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000..d992f62 --- /dev/null +++ b/opencloud/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/image-renderer-network-policy.yaml b/opencloud/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000..bcbd249 --- /dev/null +++ b/opencloud/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/image-renderer-service.yaml b/opencloud/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000..f8da127 --- /dev/null +++ b/opencloud/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/image-renderer-servicemonitor.yaml b/opencloud/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000..5d9f09d --- /dev/null +++ b/opencloud/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/ingress.yaml b/opencloud/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000..b2ffd81 --- /dev/null +++ b/opencloud/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/opencloud/charts/grafana/templates/networkpolicy.yaml b/opencloud/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000..4cd3ed6 --- /dev/null +++ b/opencloud/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/poddisruptionbudget.yaml b/opencloud/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..0525121 --- /dev/null +++ b/opencloud/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/podsecuritypolicy.yaml b/opencloud/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000..eed7af9 --- /dev/null +++ b/opencloud/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + {{- if .Values.rbac.pspUseAppArmor }} + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + {{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/opencloud/charts/grafana/templates/pvc.yaml b/opencloud/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000..d1c4b2d --- /dev/null +++ b/opencloud/charts/grafana/templates/pvc.yaml @@ -0,0 +1,39 @@ +{{- if and (not .Values.useStatefulSet) .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if and (.Values.persistence.lookupVolumeName) (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/role.yaml b/opencloud/charts/grafana/templates/role.yaml new file mode 100644 index 0000000..4b5edd9 --- /dev/null +++ b/opencloud/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.rbac.pspEnabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/rolebinding.yaml b/opencloud/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000..58f77c6 --- /dev/null +++ b/opencloud/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/route.yaml b/opencloud/charts/grafana/templates/route.yaml new file mode 100644 index 0000000..e8c98b3 --- /dev/null +++ b/opencloud/charts/grafana/templates/route.yaml @@ -0,0 +1,44 @@ +{{- range $name, $route := .Values.route }} + {{- if $route.enabled -}} +--- +apiVersion: {{ $route.apiVersion | default "gateway.networking.k8s.io/v1" }} +kind: {{ $route.kind | default "HTTPRoute" }} +metadata: + {{- with $route.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "grafana.fullname" $ }}{{ if ne $name "main" }}-{{ $name }}{{ end }} + namespace: {{ template "grafana.namespace" $ }} + labels: + app: {{ template "grafana.name" $ }}-prometheus + {{- include "grafana.labels" $ | nindent 4 }} + {{- with $route.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with $route.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $route.hostnames }} + hostnames: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if $route.additionalRules }} + {{- tpl (toYaml $route.additionalRules) $ | nindent 4 }} + {{- end }} + - backendRefs: + - name: {{ include "grafana.fullname" $ }} + port: {{ $.Values.service.port }} + {{- with $route.filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $route.matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/opencloud/charts/grafana/templates/secret-env.yaml b/opencloud/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000..eb14aac --- /dev/null +++ b/opencloud/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/secret.yaml b/opencloud/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000..fd2ca50 --- /dev/null +++ b/opencloud/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/service.yaml b/opencloud/charts/grafana/templates/service.yaml new file mode 100644 index 0000000..022328c --- /dev/null +++ b/opencloud/charts/grafana/templates/service.yaml @@ -0,0 +1,67 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- if .Values.service.ipFamilyPolicy }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + {{- end }} + {{- if .Values.service.ipFamilies }} + ipFamilies: {{ .Values.service.ipFamilies | toYaml | nindent 2 }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/serviceaccount.yaml b/opencloud/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000..ffca071 --- /dev/null +++ b/opencloud/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/servicemonitor.yaml b/opencloud/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000..0359013 --- /dev/null +++ b/opencloud/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,52 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/statefulset.yaml b/opencloud/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000..896f810 --- /dev/null +++ b/opencloud/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: storage + spec: + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test-configmap.yaml b/opencloud/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000..5695df3 --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/opencloud/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000..549400a --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test-role.yaml b/opencloud/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000..0f04c56 --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test-rolebinding.yaml b/opencloud/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000..811adb0 --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test-serviceaccount.yaml b/opencloud/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000..2e5f322 --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/opencloud/charts/grafana/templates/tests/test.yaml b/opencloud/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000..3aac160 --- /dev/null +++ b/opencloud/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": {{ .Values.testFramework.hookType | default "test" }} + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ .Values.global.imageRegistry | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/opencloud/charts/grafana/values.yaml b/opencloud/charts/grafana/values.yaml new file mode 100644 index 0000000..95696c1 --- /dev/null +++ b/opencloud/charts/grafana/values.yaml @@ -0,0 +1,1545 @@ +global: + # -- Overrides the Docker registry globally for all images + imageRegistry: null + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be templated. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: false + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: "" + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: true + ## The type of Helm hook used to run this test. Defaults to test. + ## ref: https://helm.sh/docs/topics/charts_hooks/#the-available-hooks + ## + # hookType: test + image: + # -- The Docker registry + registry: docker.io + repository: bats/bats + tag: "v1.4.1" + imagePullPolicy: IfNotPresent + securityContext: {} + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + # optional: false + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + # -- The Docker registry + registry: docker.io + repository: curlimages/curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## ConfigMap Annotations +# configMapAnnotations: {} + # argocd.argoproj.io/sync-options: Replace=true + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + # Set the ip family policy to configure dual-stack see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services) + ipFamilyPolicy: "" + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. Can be IPv4 and/or IPv6. + ipFamilies: [] + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CR is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +# -- BETA: Configure the gateway routes for the chart here. +# More routes can be added by adding a dictionary key like the 'main' route. +# Be aware that this is an early beta of this feature, +# kube-prometheus-stack does not guarantee this works and is subject to change. +# Being BETA this can/will change in the future without notice, do not use unless you want to take that risk +# [[ref]](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io%2fv1alpha2) +route: + main: + # -- Enables or disables the route + enabled: false + + # -- Set the route apiVersion, e.g. gateway.networking.k8s.io/v1 or gateway.networking.k8s.io/v1alpha2 + apiVersion: gateway.networking.k8s.io/v1 + # -- Set the route kind + # Valid options are GRPCRoute, HTTPRoute, TCPRoute, TLSRoute, UDPRoute + kind: HTTPRoute + + annotations: {} + labels: {} + + hostnames: [] + # - my-filter.example.com + parentRefs: [] + # - name: acme-gw + + matches: + - path: + type: PathPrefix + value: / + + ## Filters define the filters that are applied to requests that match this rule. + filters: [] + + ## Additional custom rules that can be added to the route + additionalRules: [] + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + disableWarning: false + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + + ## If 'lookupVolumeName' is set to true, Helm will attempt to retrieve + ## the current value of 'spec.volumeName' and incorporate it into the template. + lookupVolumeName: true + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + # -- The Docker registry + registry: docker.io + repository: library/busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # optional: false + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: https://docs.grafana.com/alerting/set-up/provision-alerting-resources/file-provisioning/ +## +alerting: {} + # policies.yaml: + # apiVersion: 1 + # policies: + # - orgId: 1 + # receiver: first_uid + # + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + # + # templates.yaml: + # apiVersion: 1 + # templates: + # - orgId: 1 + # name: my_first_template + # template: | + # {{ ` + # {{ define "my_first_template" }} + # Custom notification message + # {{ end }} + # ` }} + # + # mutetimes.yaml + # apiVersion: 1 + # muteTimes: + # - orgId: 1 + # name: mti_1 + # # refer to https://prometheus.io/docs/alerting/latest/configuration/#time_interval-0 + # time_intervals: {} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ tpl (.Values.ingress.hosts | first) . }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + # -- The Docker registry + registry: quay.io + repository: kiwigrid/k8s-sidecar + tag: 1.28.0 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # maxTotalRetries: Total number of retries to allow for any http request. + # Takes precedence over other counts. Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry. + # maxTotalRetries: 5 + # + # maxConnectRetries: How many connection-related errors to retry on for any http request. + # These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxConnectRetries: 10 + # + # maxReadRetries: How many times to retry on read errors for any http request + # These errors are raised after the request was sent to the server, so the request may have side-effects. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxReadRetries: 5 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alerts sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # + # maxTotalRetries: Total number of retries to allow for any http request. + # Takes precedence over other counts. Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry. + # maxTotalRetries: 5 + # + # maxConnectRetries: How many connection-related errors to retry on for any http request. + # These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxConnectRetries: 10 + # + # maxReadRetries: How many times to retry on read errors for any http request + # These errors are raised after the request was sent to the server, so the request may have side-effects. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxReadRetries: 5 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # folder UID. will be automatically generated if not specified + folderUid: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboards sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # maxTotalRetries: Total number of retries to allow for any http request. + # Takes precedence over other counts. Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry. + # maxTotalRetries: 5 + # + # maxConnectRetries: How many connection-related errors to retry on for any http request. + # These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxConnectRetries: 10 + # + # maxReadRetries: How many times to retry on read errors for any http request + # These errors are raised after the request was sent to the server, so the request may have side-effects. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxReadRetries: 5 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: false + # Additional datasources sidecar volume mounts + extraMounts: [] + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # maxTotalRetries: Total number of retries to allow for any http request. + # Takes precedence over other counts. Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry. + # maxTotalRetries: 5 + # + # maxConnectRetries: How many connection-related errors to retry on for any http request. + # These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxConnectRetries: 10 + # + # maxReadRetries: How many times to retry on read errors for any http request + # These errors are raised after the request was sent to the server, so the request may have side-effects. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxReadRetries: 5 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Additional plugins sidecar volume mounts + extraMounts: [] + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # maxTotalRetries: Total number of retries to allow for any http request. + # Takes precedence over other counts. Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry. + # maxTotalRetries: 5 + # + # maxConnectRetries: How many connection-related errors to retry on for any http request. + # These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxConnectRetries: 10 + # + # maxReadRetries: How many times to retry on read errors for any http request + # These errors are raised after the request was sent to the server, so the request may have side-effects. + # Applies to all requests to reloadURL and k8s api requests. + # Set to 0 to fail on the first retry of this type. + # maxReadRetries: 5 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Additional notifiers sidecar volume mounts + extraMounts: [] + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + # The url of remote image renderer if it is not in the same namespace with the grafana instance + serverURL: "" + # The callback url of grafana instances if it is not in the same namespace with the remote image renderer + renderingCallbackURL: "" + image: + # -- The Docker registry + registry: docker.io + # image-renderer Image repository + repository: grafana/grafana-image-renderer + # image-renderer Image tag + tag: latest + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + + # Extra configmaps to mount in image-renderer pods + extraConfigmapMounts: [] + + # Extra secrets to mount in image-renderer pods + extraSecretMounts: [] + + # Extra volumes to mount in image-renderer pods + extraVolumeMounts: [] + + # Extra volumes for image-renderer pods + extraVolumes: [] + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/opencloud/charts/loki/.helmignore b/opencloud/charts/loki/.helmignore new file mode 100644 index 0000000..6d90723 --- /dev/null +++ b/opencloud/charts/loki/.helmignore @@ -0,0 +1,32 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ + +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ + +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Other +doc.yaml +README.tpl +README.md.gotmpl +ci +CHANGELOG.md diff --git a/opencloud/charts/loki/Chart.lock b/opencloud/charts/loki/Chart.lock new file mode 100644 index 0000000..e4ffe8c --- /dev/null +++ b/opencloud/charts/loki/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: minio + repository: https://charts.min.io/ + version: 5.3.0 +- name: grafana-agent-operator + repository: https://grafana.github.io/helm-charts + version: 0.5.0 +- name: rollout-operator + repository: https://grafana.github.io/helm-charts + version: 0.20.0 +digest: sha256:7d7078dd5a08a25d152fd78a6fe60a8795cff0f940507d5b4c7ad4df396a16d9 +generated: "2024-11-28T15:43:12.190731467+01:00" diff --git a/opencloud/charts/loki/Chart.yaml b/opencloud/charts/loki/Chart.yaml new file mode 100644 index 0000000..8224157 --- /dev/null +++ b/opencloud/charts/loki/Chart.yaml @@ -0,0 +1,32 @@ +apiVersion: v2 +appVersion: 3.3.1 +dependencies: +- alias: minio + condition: minio.enabled + name: minio + repository: https://charts.min.io/ + version: 5.3.0 +- alias: grafana-agent-operator + condition: monitoring.selfMonitoring.grafanaAgent.installOperator + name: grafana-agent-operator + repository: https://grafana.github.io/helm-charts + version: 0.5.0 +- alias: rollout_operator + condition: rollout_operator.enabled + name: rollout-operator + repository: https://grafana.github.io/helm-charts + version: 0.20.0 +description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both + simple, scalable and distributed modes. +home: https://grafana.github.io/helm-charts +icon: https://grafana.com/docs/loki/latest/logo_and_name.png +maintainers: +- name: trevorwhitney +- name: jeschkies +name: loki +sources: +- https://github.com/grafana/loki +- https://grafana.com/oss/loki/ +- https://grafana.com/docs/loki/latest/ +type: application +version: 6.23.0 diff --git a/opencloud/charts/loki/Makefile b/opencloud/charts/loki/Makefile new file mode 100644 index 0000000..4b56414 --- /dev/null +++ b/opencloud/charts/loki/Makefile @@ -0,0 +1,7 @@ +.DEFAULT_GOAL := all +.PHONY: lint lint-yaml + +lint: lint-yaml + +lint-yaml: + yamllint -c $(CURDIR)/src/.yamllint.yaml $(CURDIR)/src diff --git a/opencloud/charts/loki/README.md b/opencloud/charts/loki/README.md new file mode 100644 index 0000000..bbc39ed --- /dev/null +++ b/opencloud/charts/loki/README.md @@ -0,0 +1,65 @@ +# loki + +![Version: 6.23.0](https://img.shields.io/badge/Version-6.23.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.3.1](https://img.shields.io/badge/AppVersion-3.3.1-informational?style=flat-square) + +Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. + +## Source Code + +* +* +* + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.min.io/ | minio(minio) | 5.3.0 | +| https://grafana.github.io/helm-charts | grafana-agent-operator(grafana-agent-operator) | 0.5.0 | +| https://grafana.github.io/helm-charts | rollout_operator(rollout-operator) | 0.20.0 | + +Find more information in the Loki Helm Chart [documentation](https://grafana.com/docs/loki/next/installation/helm). + +## Contributing and releasing + +If you made any changes to the [Chart.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml) or [values.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml) run `make helm-docs` from the root of the repository to update the documentation and commit the changed files. + +Futhermore, please add an entry to the [CHANGELOG.md](./CHANGELOG.md) file about what you changed. This file has a header that looks like this: + +``` +[//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +```` + +Place your changes as a bulleted list below this header. The helm chart is automatically released once a week, at which point the `CHANGELOG.md` file will be updated to reflect the release of all changes between this header the the header of the previous version as the changes for that weeks release. For example, if the weekly release will be `1.21.0`, and the `CHANGELOG.md` file has the following entries: + +``` +[//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) + +- [CHANGE] Changed the thing +- [FEATURE] Cool new feature + +## 1.20.0 + +- [BUGFIX] Fixed the bug +``` + +Then the weekly release will create a `CHANGELOG.md` with the following content: +``` +[//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) + +## 1.21.0 + +- [CHANGE] Changed the thing +- [FEATURE] Cool new feature + +## 1.20.0 + +- [BUGFIX] Fixed the bug +``` + +#### Versioning + +Normally contributors need _not_ bump the version nor update the [CHANGELOG.md](https://github.com/grafana/loki/blob/main/production/helm/loki/CHANGELOG.md). A new version of the Chart will follow this cadence: +- Automatic weekly releases +- Releases that coincide with Loki/GEL releases +- Manual releases when necessary (ie. to address a CVE or critical bug) diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/.helmignore b/opencloud/charts/loki/charts/grafana-agent-operator/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/Chart.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/Chart.yaml new file mode 100644 index 0000000..3281eec --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +appVersion: 0.43.3 +description: A Helm chart for Grafana Agent Operator +home: https://grafana.com/docs/agent/v0.43/ +icon: https://raw.githubusercontent.com/grafana/agent/v0.43.3/docs/sources/assets/logo_and_name.png +maintainers: +- email: grafana-agent-team@googlegroups.com + name: Grafana Agent Team +name: grafana-agent-operator +sources: +- https://github.com/grafana/agent/tree/v0.43.3/static/operator +type: application +version: 0.5.0 diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/README.md b/opencloud/charts/loki/charts/grafana-agent-operator/README.md new file mode 100644 index 0000000..b39b26a --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/README.md @@ -0,0 +1,82 @@ +# grafana-agent-operator + +![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.43.3](https://img.shields.io/badge/AppVersion-0.43.3-informational?style=flat-square) + +A Helm chart for Grafana Agent Operator + +⚠️ **Please create issues relating to this Helm chart in the [Agent](https://github.com/grafana/agent/issues) repo.** + +## Source Code + +* + +Note that this chart does not provision custom resources like `GrafanaAgent` and `MetricsInstance` (formerly `PrometheusInstance`) or any `*Monitor` resources. + +To learn how to deploy these resources, please see Grafana's [Agent Operator getting started guide](https://grafana.com/docs/agent/latest/operator/getting-started/). + +## CRDs + +The CRDs are synced into this chart manually (for now) from the Grafana Agent [GitHub repo](https://github.com/grafana/agent/tree/main/operations/agent-static-operator/crds). To learn more about how Helm manages CRDs, please see [Custom Resource Definitions](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) from the Helm docs. + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana-agent-operator +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. Until this chart's version reaches `v1.0`, there are no promises of backwards compatibility. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Pod affinity configuration | +| annotations | object | `{}` | Annotations for the Deployment | +| containerSecurityContext | object | `{}` | Container security context (allowPrivilegeEscalation, etc.) | +| extraArgs | list | `[]` | List of additional cli arguments to configure agent-operator (example: `--log.level`) | +| fullnameOverride | string | `""` | Overrides the chart's computed fullname | +| global.commonLabels | object | `{}` | Common labels for all object directly managed by this chart. | +| hostAliases | list | `[]` | hostAliases to add | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Image pull secrets | +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"grafana/agent-operator"` | Image repo | +| image.tag | string | `"v0.43.3"` | Image tag | +| kubeletService | object | `{"namespace":"default","serviceName":"kubelet"}` | If both are set, Agent Operator will create and maintain a service for scraping kubelets https://grafana.com/docs/agent/latest/operator/getting-started/#monitor-kubelets | +| nameOverride | string | `""` | Overrides the chart's name | +| nodeSelector | object | `{}` | nodeSelector configuration | +| podAnnotations | object | `{}` | Annotations for the Deployment Pods | +| podLabels | object | `{}` | Annotations for the Deployment Pods | +| podSecurityContext | object | `{}` | Pod security context (runAsUser, etc.) | +| rbac.create | bool | `true` | Toggle to create ClusterRole and ClusterRoleBinding | +| rbac.podSecurityPolicyName | string | `""` | Name of a PodSecurityPolicy to use in the ClusterRole. If unset, no PodSecurityPolicy is used. | +| resources | object | `{}` | Resource limits and requests config | +| serviceAccount.create | bool | `true` | Toggle to create ServiceAccount | +| serviceAccount.name | string | `nil` | Service account name | +| test.image.registry | string | `"docker.io"` | Test image registry | +| test.image.repository | string | `"library/busybox"` | Test image repo | +| test.image.tag | string | `"latest"` | Test image tag | +| tolerations | list | `[]` | Tolerations applied to Pods | diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/README.md.gotmpl b/opencloud/charts/loki/charts/grafana-agent-operator/README.md.gotmpl new file mode 100644 index 0000000..3dce97a --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/README.md.gotmpl @@ -0,0 +1,52 @@ +{{ template "chart.header" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +⚠️ **Please create issues relating to this Helm chart in the [Agent](https://github.com/grafana/agent/issues) repo.** + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +Note that this chart does not provision custom resources like `GrafanaAgent` and `MetricsInstance` (formerly `PrometheusInstance`) or any `*Monitor` resources. + +To learn how to deploy these resources, please see Grafana's [Agent Operator getting started guide](https://grafana.com/docs/agent/latest/operator/getting-started/). + +## CRDs + +The CRDs are synced into this chart manually (for now) from the Grafana Agent [GitHub repo](https://github.com/grafana/agent/tree/main/operations/agent-static-operator/crds). To learn more about how Helm manages CRDs, please see [Custom Resource Definitions](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) from the Helm docs. + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana-agent-operator +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. Until this chart's version reaches `v1.0`, there are no promises of backwards compatibility. + +{{ template "chart.valuesSection" . }} diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_podmonitors.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_podmonitors.yaml new file mode 100644 index 0000000..153677b --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_podmonitors.yaml @@ -0,0 +1,424 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: podmonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: PodMonitor + listKind: PodMonitorList + plural: podmonitors + shortNames: + - pmon + singular: podmonitor + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + attachMetadata: + properties: + node: + type: boolean + type: object + jobLabel: + type: string + labelLimit: + format: int64 + type: integer + labelNameLengthLimit: + format: int64 + type: integer + labelValueLengthLimit: + format: int64 + type: integer + namespaceSelector: + properties: + any: + type: boolean + matchNames: + items: + type: string + type: array + type: object + podMetricsEndpoints: + items: + properties: + authorization: + properties: + credentials: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + enableHttp2: + type: boolean + filterRunning: + type: boolean + followRedirects: + type: boolean + honorLabels: + type: boolean + honorTimestamps: + type: boolean + interval: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + params: + additionalProperties: + items: + type: string + type: array + type: object + path: + type: string + port: + type: string + proxyUrl: + type: string + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + enum: + - http + - https + type: string + scrapeTimeout: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + type: boolean + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + type: array + podTargetLabels: + items: + type: string + type: array + sampleLimit: + format: int64 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + targetLimit: + format: int64 + type: integer + required: + - podMetricsEndpoints + - selector + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_probes.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_probes.yaml new file mode 100644 index 0000000..13fc36f --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_probes.yaml @@ -0,0 +1,458 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: probes.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: Probe + listKind: ProbeList + plural: probes + shortNames: + - prb + singular: probe + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + authorization: + properties: + credentials: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + interval: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + jobName: + type: string + labelLimit: + format: int64 + type: integer + labelNameLengthLimit: + format: int64 + type: integer + labelValueLengthLimit: + format: int64 + type: integer + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + module: + type: string + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + prober: + properties: + path: + default: /probe + type: string + proxyUrl: + type: string + scheme: + enum: + - http + - https + type: string + url: + type: string + required: + - url + type: object + sampleLimit: + format: int64 + type: integer + scrapeTimeout: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + targetLimit: + format: int64 + type: integer + targets: + properties: + ingress: + properties: + namespaceSelector: + properties: + any: + type: boolean + matchNames: + items: + type: string + type: array + type: object + relabelingConfigs: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + type: object + staticConfig: + properties: + labels: + additionalProperties: + type: string + type: object + relabelingConfigs: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + static: + items: + type: string + type: array + type: object + type: object + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + type: boolean + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_servicemonitors.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_servicemonitors.yaml new file mode 100644 index 0000000..ff62f8f --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.coreos.com_servicemonitors.yaml @@ -0,0 +1,436 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: servicemonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: ServiceMonitor + listKind: ServiceMonitorList + plural: servicemonitors + shortNames: + - smon + singular: servicemonitor + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + attachMetadata: + properties: + node: + type: boolean + type: object + endpoints: + items: + properties: + authorization: + properties: + credentials: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenFile: + type: string + bearerTokenSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + enableHttp2: + type: boolean + filterRunning: + type: boolean + followRedirects: + type: boolean + honorLabels: + type: boolean + honorTimestamps: + type: boolean + interval: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + params: + additionalProperties: + items: + type: string + type: array + type: object + path: + type: string + port: + type: string + proxyUrl: + type: string + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + enum: + - http + - https + type: string + scrapeTimeout: + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + type: array + jobLabel: + type: string + labelLimit: + format: int64 + type: integer + labelNameLengthLimit: + format: int64 + type: integer + labelValueLengthLimit: + format: int64 + type: integer + namespaceSelector: + properties: + any: + type: boolean + matchNames: + items: + type: string + type: array + type: object + podTargetLabels: + items: + type: string + type: array + sampleLimit: + format: int64 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + targetLabels: + items: + type: string + type: array + targetLimit: + format: int64 + type: integer + required: + - endpoints + - selector + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_grafanaagents.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_grafanaagents.yaml new file mode 100644 index 0000000..4ec31d6 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_grafanaagents.yaml @@ -0,0 +1,3711 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: grafanaagents.monitoring.grafana.com +spec: + group: monitoring.grafana.com + names: + categories: + - agent-operator + kind: GrafanaAgent + listKind: GrafanaAgentList + plural: grafanaagents + singular: grafanaagent + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + apiServer: + properties: + authorization: + properties: + credentials: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + credentialsFile: + type: string + type: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerToken: + type: string + bearerTokenFile: + type: string + host: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + required: + - host + type: object + configMaps: + items: + type: string + type: array + configReloaderImage: + type: string + configReloaderVersion: + type: string + containers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + disableReporting: + default: false + type: boolean + disableSupportBundle: + default: false + type: boolean + enableConfigReadAPI: + default: false + type: boolean + image: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + integrations: + properties: + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + type: object + logFormat: + type: string + logLevel: + type: string + logs: + properties: + clients: + items: + properties: + backoffConfig: + properties: + maxPeriod: + type: string + maxRetries: + type: integer + minPeriod: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + batchSize: + type: integer + batchWait: + type: string + bearerToken: + type: string + bearerTokenFile: + type: string + externalLabels: + additionalProperties: + type: string + type: object + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + proxyUrl: + type: string + tenantId: + type: string + timeout: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + url: + type: string + required: + - url + type: object + type: array + enforcedNamespaceLabel: + type: string + ignoreNamespaceSelectors: + type: boolean + instanceNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + instanceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + logsExternalLabelName: + type: string + type: object + metrics: + properties: + arbitraryFSAccessThroughSMs: + properties: + deny: + type: boolean + type: object + enforcedNamespaceLabel: + type: string + enforcedSampleLimit: + format: int64 + type: integer + enforcedTargetLimit: + format: int64 + type: integer + externalLabels: + additionalProperties: + type: string + type: object + ignoreNamespaceSelectors: + type: boolean + instanceNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + instanceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + metricsExternalLabelName: + type: string + overrideHonorLabels: + type: boolean + overrideHonorTimestamps: + type: boolean + remoteWrite: + items: + properties: + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerToken: + type: string + bearerTokenFile: + type: string + headers: + additionalProperties: + type: string + type: object + metadataConfig: + properties: + send: + type: boolean + sendInterval: + type: string + type: object + name: + type: string + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + proxyUrl: + type: string + queueConfig: + properties: + batchSendDeadline: + type: string + capacity: + type: integer + maxBackoff: + type: string + maxRetries: + type: integer + maxSamplesPerSend: + type: integer + maxShards: + type: integer + minBackoff: + type: string + minShards: + type: integer + retryOnRateLimit: + type: boolean + type: object + remoteTimeout: + type: string + sigv4: + properties: + accessKey: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + profile: + type: string + region: + type: string + roleARN: + type: string + secretKey: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + url: + type: string + writeRelabelConfigs: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + required: + - url + type: object + type: array + replicaExternalLabelName: + type: string + replicas: + format: int32 + type: integer + scrapeInterval: + type: string + scrapeTimeout: + type: string + shards: + format: int32 + type: integer + type: object + nodeSelector: + additionalProperties: + type: string + type: object + paused: + type: boolean + podMetadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: object + portName: + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + runtimeClassName: + type: string + secrets: + items: + type: string + type: array + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + storage: + properties: + disableMountSubPath: + type: boolean + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + volumeClaimTemplate: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + allocatedResourceStatuses: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: granular + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + version: + type: string + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_integrations.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_integrations.yaml new file mode 100644 index 0000000..960b2f7 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_integrations.yaml @@ -0,0 +1,810 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: integrations.monitoring.grafana.com +spec: + group: monitoring.grafana.com + names: + categories: + - agent-operator + kind: Integration + listKind: IntegrationList + plural: integrations + singular: integration + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + config: + type: object + x-kubernetes-preserve-unknown-fields: true + configMaps: + items: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + name: + type: string + secrets: + items: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + type: + properties: + allNodes: + type: boolean + unique: + type: boolean + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - config + - name + - type + type: object + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_logsinstances.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_logsinstances.yaml new file mode 100644 index 0000000..517bb30 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_logsinstances.yaml @@ -0,0 +1,299 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: logsinstances.monitoring.grafana.com +spec: + group: monitoring.grafana.com + names: + categories: + - agent-operator + kind: LogsInstance + listKind: LogsInstanceList + plural: logsinstances + singular: logsinstance + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + additionalScrapeConfigs: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + clients: + items: + properties: + backoffConfig: + properties: + maxPeriod: + type: string + maxRetries: + type: integer + minPeriod: + type: string + type: object + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + batchSize: + type: integer + batchWait: + type: string + bearerToken: + type: string + bearerTokenFile: + type: string + externalLabels: + additionalProperties: + type: string + type: object + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + proxyUrl: + type: string + tenantId: + type: string + timeout: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + url: + type: string + required: + - url + type: object + type: array + podLogsNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + podLogsSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + targetConfig: + properties: + syncPeriod: + type: string + type: object + type: object + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_metricsinstances.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_metricsinstances.yaml new file mode 100644 index 0000000..610193f --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_metricsinstances.yaml @@ -0,0 +1,495 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: metricsinstances.monitoring.grafana.com +spec: + group: monitoring.grafana.com + names: + categories: + - agent-operator + kind: MetricsInstance + listKind: MetricsInstanceList + plural: metricsinstances + singular: metricsinstance + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + additionalScrapeConfigs: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + maxWALTime: + type: string + minWALTime: + type: string + podMonitorNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + podMonitorSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + probeNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + probeSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + remoteFlushDeadline: + type: string + remoteWrite: + items: + properties: + basicAuth: + properties: + password: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerToken: + type: string + bearerTokenFile: + type: string + headers: + additionalProperties: + type: string + type: object + metadataConfig: + properties: + send: + type: boolean + sendInterval: + type: string + type: object + name: + type: string + oauth2: + properties: + clientId: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + type: object + scopes: + items: + type: string + type: array + tokenUrl: + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + proxyUrl: + type: string + queueConfig: + properties: + batchSendDeadline: + type: string + capacity: + type: integer + maxBackoff: + type: string + maxRetries: + type: integer + maxSamplesPerSend: + type: integer + maxShards: + type: integer + minBackoff: + type: string + minShards: + type: integer + retryOnRateLimit: + type: boolean + type: object + remoteTimeout: + type: string + sigv4: + properties: + accessKey: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + profile: + type: string + region: + type: string + roleARN: + type: string + secretKey: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + url: + type: string + writeRelabelConfigs: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + required: + - url + type: object + type: array + serviceMonitorNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + serviceMonitorSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + walTruncateFrequency: + type: string + writeStaleOnShutdown: + type: boolean + type: object + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_podlogs.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_podlogs.yaml new file mode 100644 index 0000000..f22d051 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/crds/monitoring.grafana.com_podlogs.yaml @@ -0,0 +1,308 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: podlogs.monitoring.grafana.com +spec: + group: monitoring.grafana.com + names: + categories: + - agent-operator + kind: PodLogs + listKind: PodLogsList + plural: podlogs + singular: podlogs + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + jobLabel: + type: string + namespaceSelector: + properties: + any: + type: boolean + matchNames: + items: + type: string + type: array + type: object + pipelineStages: + items: + properties: + cri: + type: object + docker: + type: object + drop: + properties: + dropCounterReason: + type: string + expression: + type: string + longerThan: + type: string + olderThan: + type: string + source: + type: string + value: + type: string + type: object + json: + properties: + expressions: + additionalProperties: + type: string + type: object + source: + type: string + type: object + labelAllow: + items: + type: string + type: array + labelDrop: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + limit: + properties: + burst: + type: integer + drop: + type: boolean + rate: + type: integer + type: object + match: + properties: + action: + type: string + dropCounterReason: + type: string + pipelineName: + type: string + selector: + type: string + stages: + type: string + required: + - selector + type: object + metrics: + additionalProperties: + properties: + action: + type: string + buckets: + items: + type: string + type: array + countEntryBytes: + type: boolean + description: + type: string + matchAll: + type: boolean + maxIdleDuration: + type: string + prefix: + type: string + source: + type: string + type: + type: string + value: + type: string + required: + - action + - type + type: object + type: object + multiline: + properties: + firstLine: + type: string + maxLines: + type: integer + maxWaitTime: + type: string + required: + - firstLine + type: object + output: + properties: + source: + type: string + required: + - source + type: object + pack: + properties: + ingestTimestamp: + type: boolean + labels: + items: + type: string + type: array + required: + - labels + type: object + regex: + properties: + expression: + type: string + source: + type: string + required: + - expression + type: object + replace: + properties: + expression: + type: string + replace: + type: string + source: + type: string + required: + - expression + type: object + template: + properties: + source: + type: string + template: + type: string + required: + - source + - template + type: object + tenant: + properties: + label: + type: string + source: + type: string + value: + type: string + type: object + timestamp: + properties: + actionOnFailure: + type: string + fallbackFormats: + items: + type: string + type: array + format: + type: string + location: + type: string + source: + type: string + required: + - format + - source + type: object + type: object + type: array + podTargetLabels: + items: + type: string + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - selector + type: object + type: object + served: true + storage: true diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/_helpers.tpl b/opencloud/charts/loki/charts/grafana-agent-operator/templates/_helpers.tpl new file mode 100644 index 0000000..e374499 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/_helpers.tpl @@ -0,0 +1,70 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ga-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ga-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ga-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ga-operator.labels" -}} +{{ include "ga-operator.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: operator +helm.sh/chart: {{ include "ga-operator.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- with .Values.global.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ga-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ga-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ga-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ga-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrole.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrole.yaml new file mode 100644 index 0000000..08ad58c --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrole.yaml @@ -0,0 +1,62 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ga-operator.fullname" . }} + labels: +{{ include "ga-operator.labels" . | indent 4 }} +rules: +- apiGroups: [monitoring.grafana.com] + resources: + - grafanaagents + - metricsinstances + - logsinstances + - podlogs + - integrations + verbs: [get, list, watch] +- apiGroups: [monitoring.grafana.com] + resources: + - grafanaagents/finalizers + - metricsinstances/finalizers + - logsinstances/finalizers + - podlogs/finalizers + - integrations/finalizers + verbs: [get, list, watch, update] +- apiGroups: [monitoring.coreos.com] + resources: + - podmonitors + - probes + - servicemonitors + verbs: [get, list, watch] +- apiGroups: [monitoring.coreos.com] + resources: + - podmonitors/finalizers + - probes/finalizers + - servicemonitors/finalizers + verbs: [get, list, watch, update] +- apiGroups: [""] + resources: + - namespaces + - nodes + verbs: [get, list, watch] +- apiGroups: [""] + resources: + - secrets + - services + - configmaps + - endpoints + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: ["apps"] + resources: + - statefulsets + - daemonsets + - deployments + verbs: [get, list, watch, create, update, patch, delete] +{{- with .Values.rbac.podSecurityPolicyName }} +- apiGroups: [policy] + resources: + - podsecuritypolicies + verbs: [use] + resourceNames: [ {{ . }} ] +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrolebinding.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrolebinding.yaml new file mode 100644 index 0000000..372d310 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ga-operator.fullname" . }} + labels: +{{ include "ga-operator.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ga-operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "ga-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} + diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-deployment.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-deployment.yaml new file mode 100644 index 0000000..3367db1 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ga-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: +{{ include "ga-operator.labels" . | indent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: +{{ include "ga-operator.selectorLabels" . | indent 6 }} + template: + metadata: + labels: +{{ include "ga-operator.selectorLabels" . | indent 8 }} +{{- with .Values.podLabels }} +{{- toYaml . | nindent 8 }} +{{- end }} +{{- with .Values.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + serviceAccountName: {{ template "ga-operator.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ include "ga-operator.name" . }} + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if or (and .Values.kubeletService.namespace .Values.kubeletService.serviceName) (.Values.extraArgs) }} + args: + {{- if and .Values.kubeletService.namespace .Values.kubeletService.serviceName }} + - --kubelet-service={{ .Values.kubeletService.namespace }}/{{ .Values.kubeletService.serviceName }} + {{- end }} + {{- if .Values.extraArgs }} + {{- range .Values.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-serviceaccount.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-serviceaccount.yaml new file mode 100644 index 0000000..1f9b207 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/operator-serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "ga-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: +{{ include "ga-operator.labels" . | indent 4 }} +{{- end -}} + diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/templates/tests/test-grafanaagent.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/templates/tests/test-grafanaagent.yaml new file mode 100644 index 0000000..4001da4 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/templates/tests/test-grafanaagent.yaml @@ -0,0 +1,118 @@ +apiVersion: monitoring.grafana.com/v1alpha1 +kind: GrafanaAgent +metadata: + name: grafana-agent-test + labels: + app: grafana-agent-test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +spec: + image: "{{ .Values.image.registry }}/grafana/agent:{{ .Values.image.tag }}" + logLevel: info + serviceAccountName: grafana-agent-test-sa + metrics: + instanceSelector: + matchLabels: + agent: grafana-agent-test + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-agent-test-sa + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: grafana-agent-test-cr + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- nonResourceURLs: + - /metrics + - /metrics/cadvisor + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: grafana-agent-test-crb + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-agent-test-cr +subjects: +- kind: ServiceAccount + name: grafana-agent-test-sa + namespace: {{ .Release.Namespace }} + +--- + +apiVersion: monitoring.grafana.com/v1alpha1 +kind: MetricsInstance +metadata: + name: primary-test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + labels: + agent: grafana-agent-test +spec: {} + +--- + +apiVersion: v1 +kind: Pod +metadata: + name: grafana-agent-test-probe + annotations: + "helm.sh/hook": test + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +spec: + containers: + - name: busybox + image: "{{ .Values.test.image.registry }}/{{ .Values.test.image.repository }}:{{ .Values.test.image.tag }}" + command: ['wget'] + args: ['grafana-agent-test-operated:8080/-/healthy'] + # Wait for GrafanaAgent CR + initContainers: + - name: sleep + image: "{{ .Values.test.image.registry }}/{{ .Values.test.image.repository }}:{{ .Values.test.image.tag }}" + command: ['sleep', '60'] + restartPolicy: Never diff --git a/opencloud/charts/loki/charts/grafana-agent-operator/values.yaml b/opencloud/charts/loki/charts/grafana-agent-operator/values.yaml new file mode 100644 index 0000000..5f7adc5 --- /dev/null +++ b/opencloud/charts/loki/charts/grafana-agent-operator/values.yaml @@ -0,0 +1,84 @@ +global: + # -- Common labels for all object directly managed by this chart. + commonLabels: {} + +# -- Overrides the chart's name +nameOverride: "" + +# -- Overrides the chart's computed fullname +fullnameOverride: "" + +# -- Annotations for the Deployment +annotations: {} + +# -- Annotations for the Deployment Pods +podAnnotations: {} + +# -- Annotations for the Deployment Pods +podLabels: {} + +# -- Pod security context (runAsUser, etc.) +podSecurityContext: {} + +# -- Container security context (allowPrivilegeEscalation, etc.) +containerSecurityContext: {} + +rbac: + # -- Toggle to create ClusterRole and ClusterRoleBinding + create: true + # -- Name of a PodSecurityPolicy to use in the ClusterRole. If unset, no PodSecurityPolicy is used. + podSecurityPolicyName: '' + +serviceAccount: + # -- Toggle to create ServiceAccount + create: true + # -- Service account name + name: + +image: + # -- Image registry + registry: docker.io + # -- Image repo + repository: grafana/agent-operator + # -- Image tag + tag: v0.43.3 + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Image pull secrets + pullSecrets: [] + +test: + image: + # -- Test image registry + registry: docker.io + # -- Test image repo + repository: library/busybox + # -- Test image tag + tag: latest + +# -- hostAliases to add +hostAliases: [] +# - ip: 1.2.3.4 +# hostnames: +# - domain.tld + +# -- If both are set, Agent Operator will create and maintain a service for scraping kubelets +# https://grafana.com/docs/agent/latest/operator/getting-started/#monitor-kubelets +kubeletService: + namespace: default + serviceName: kubelet + +# -- List of additional cli arguments to configure agent-operator (example: `--log.level`) +extraArgs: [] + +# -- Resource limits and requests config +resources: {} + +# -- nodeSelector configuration +nodeSelector: {} + +# -- Tolerations applied to Pods +tolerations: [] + +# -- Pod affinity configuration +affinity: {} diff --git a/opencloud/charts/loki/charts/minio/.helmignore b/opencloud/charts/loki/charts/minio/.helmignore new file mode 100644 index 0000000..a9fe727 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# OWNERS file for Kubernetes +OWNERS \ No newline at end of file diff --git a/opencloud/charts/loki/charts/minio/Chart.yaml b/opencloud/charts/loki/charts/minio/Chart.yaml new file mode 100644 index 0000000..10a6423 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +appVersion: RELEASE.2024-04-18T19-09-19Z +description: High Performance Object Storage +home: https://min.io +icon: https://min.io/resources/img/logo/MINIO_wordmark.png +keywords: +- minio +- storage +- object-storage +- s3 +- cluster +maintainers: +- email: dev@minio.io + name: MinIO, Inc +name: minio +sources: +- https://github.com/minio/minio +version: 5.3.0 diff --git a/opencloud/charts/loki/charts/minio/README.md b/opencloud/charts/loki/charts/minio/README.md new file mode 100644 index 0000000..a1d5c99 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/README.md @@ -0,0 +1,264 @@ +# MinIO Community Helm Chart + +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/minio/blob/master/LICENSE) + +MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. + +| IMPORTANT | +| -------------------------- | +| This Helm chart is community built, maintained, and supported. MinIO does not guarantee support for any given bug, feature request, or update referencing this chart.

MinIO publishes a separate [MinIO Kubernetes Operator and Tenant Helm Chart](https://github.com/minio/operator/tree/master/helm) that is officially maintained and supported. MinIO strongly recommends using the MinIO Kubernetes Operator for production deployments. See [Deploy Operator With Helm](https://min.io/docs/minio/kubernetes/upstream/operations/install-deploy-manage/deploy-operator-helm.html?ref=github) for additional documentation. | + +## Introduction + +This chart bootstraps MinIO Cluster on [Kubernetes](http://kubernetes.io) using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm cli with Kubernetes cluster configured. +- PV provisioner support in the underlying infrastructure. (We recommend using ) +- Use Kubernetes version v1.19 and later for best experience. + +## Configure MinIO Helm repo + +```bash +helm repo add minio https://charts.min.io/ +``` + +### Installing the Chart + +Install this chart using: + +```bash +helm install --namespace minio --set rootUser=rootuser,rootPassword=rootpass123 --generate-name minio/minio +``` + +The command deploys MinIO on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +### Installing the Chart (toy-setup) + +Minimal toy setup for testing purposes can be deployed using: + +```bash +helm install --set resources.requests.memory=512Mi --set replicas=1 --set persistence.enabled=false --set mode=standalone --set rootUser=rootuser,rootPassword=rootpass123 --generate-name minio/minio +``` + +### Upgrading the Chart + +You can use Helm to update MinIO version in a live release. Assuming your release is named as `my-release`, get the values using the command: + +```bash +helm get values my-release > old_values.yaml +``` + +Then change the field `image.tag` in `old_values.yaml` file with MinIO image tag you want to use. Now update the chart using + +```bash +helm upgrade -f old_values.yaml my-release minio/minio +``` + +Default upgrade strategies are specified in the `values.yaml` file. Update these fields if you'd like to use a different strategy. + +### Configuration + +Refer the [Values file](./values.yaml) for all the possible config fields. + +You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```bash +helm install --name my-release --set persistence.size=1Ti minio/minio +``` + +The above command deploys MinIO server with a 1Ti backing persistent volume. + +Alternately, you can provide a YAML file that specifies parameter values while installing the chart. For example, + +```bash +helm install --name my-release -f values.yaml minio/minio +``` + +### Persistence + +This chart provisions a PersistentVolumeClaim and mounts corresponding persistent volume to default location `/export`. You'll need physical storage available in the Kubernetes cluster for this to work. If you'd rather use `emptyDir`, disable PersistentVolumeClaim by: + +```bash +helm install --set persistence.enabled=false minio/minio +``` + +> *"An emptyDir volume is first created when a Pod is assigned to a Node, and exists as long as that Pod is running on that node. When a Pod is removed from a node for any reason, the data in the emptyDir is deleted forever."* + +### Existing PersistentVolumeClaim + +If a Persistent Volume Claim already exists, specify it during installation. + +1. Create the PersistentVolume +2. Create the PersistentVolumeClaim +3. Install the chart + +```bash +helm install --set persistence.existingClaim=PVC_NAME minio/minio +``` + +### NetworkPolicy + +To enable network policy for MinIO, +install [a networking plugin that implements the Kubernetes +NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), +and set `networkPolicy.enabled` to `true`. + +For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting +the DefaultDeny namespace annotation. Note: this will enforce policy for *all* pods in the namespace: + +``` +kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" +``` + +When using `Cilium` as a CNI in your cluster, please edit the `flavor` field to `cilium`. + +With NetworkPolicy enabled, traffic will be limited to just port 9000. + +For more precise policy, set `networkPolicy.allowExternal=true`. This will +only allow pods with the generated client label to connect to MinIO. +This label will be displayed in the output of a successful install. + +### Existing secret + +Instead of having this chart create the secret for you, you can supply a preexisting secret, much +like an existing PersistentVolumeClaim. + +First, create the secret: + +```bash +kubectl create secret generic my-minio-secret --from-literal=rootUser=foobarbaz --from-literal=rootPassword=foobarbazqux +``` + +Then install the chart, specifying that you want to use an existing secret: + +```bash +helm install --set existingSecret=my-minio-secret minio/minio +``` + +The following fields are expected in the secret: + +| .data.\ in Secret | Corresponding variable | Description | Required | +|:------------------------|:-----------------------|:---------------|:---------| +| `rootUser` | `rootUser` | Root user. | yes | +| `rootPassword` | `rootPassword` | Root password. | yes | + +All corresponding variables will be ignored in values file. + +### Configure TLS + +To enable TLS for MinIO containers, acquire TLS certificates from a CA or create self-signed certificates. While creating / acquiring certificates ensure the corresponding domain names are set as per the standard [DNS naming conventions](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity) in a Kubernetes StatefulSet (for a distributed MinIO setup). Then create a secret using + +```bash +kubectl create secret generic tls-ssl-minio --from-file=path/to/private.key --from-file=path/to/public.crt +``` + +Then install the chart, specifying that you want to use the TLS secret: + +```bash +helm install --set tls.enabled=true,tls.certSecret=tls-ssl-minio minio/minio +``` + +### Installing certificates from third party CAs + +MinIO can connect to other servers, including MinIO nodes or other server types such as NATs and Redis. If these servers use certificates that were not registered with a known CA, add trust for these certificates to MinIO Server by bundling these certificates into a Kubernetes secret and providing it to Helm via the `trustedCertsSecret` value. If `.Values.tls.enabled` is `true` and you're installing certificates for third party CAs, remember to include MinIO's own certificate with key `public.crt`, if it also needs to be trusted. + +For instance, given that TLS is enabled and you need to add trust for MinIO's own CA and for the CA of a Keycloak server, a Kubernetes secret can be created from the certificate files using `kubectl`: + +``` +kubectl -n minio create secret generic minio-trusted-certs --from-file=public.crt --from-file=keycloak.crt +``` + +If TLS is not enabled, you would need only the third party CA: + +``` +kubectl -n minio create secret generic minio-trusted-certs --from-file=keycloak.crt +``` + +The name of the generated secret can then be passed to Helm using a values file or the `--set` parameter: + +``` +trustedCertsSecret: "minio-trusted-certs" + +or + +--set trustedCertsSecret=minio-trusted-certs +``` + +### Create buckets after install + +Install the chart, specifying the buckets you want to create after install: + +```bash +helm install --set buckets[0].name=bucket1,buckets[0].policy=none,buckets[0].purge=false minio/minio +``` + +Description of the configuration parameters used above - + +- `buckets[].name` - name of the bucket to create, must be a string with length > 0 +- `buckets[].policy` - can be one of none|download|upload|public +- `buckets[].purge` - purge if bucket exists already + +### Create policies after install + +Install the chart, specifying the policies you want to create after install: + +```bash +helm install --set policies[0].name=mypolicy,policies[0].statements[0].resources[0]='arn:aws:s3:::bucket1',policies[0].statements[0].actions[0]='s3:ListBucket',policies[0].statements[0].actions[1]='s3:GetObject' minio/minio +``` + +Description of the configuration parameters used above - + +- `policies[].name` - name of the policy to create, must be a string with length > 0 +- `policies[].statements[]` - list of statements, includes actions and resources +- `policies[].statements[].resources[]` - list of resources that applies the statement +- `policies[].statements[].actions[]` - list of actions granted + +### Create user after install + +Install the chart, specifying the users you want to create after install: + +```bash +helm install --set users[0].accessKey=accessKey,users[0].secretKey=secretKey,users[0].policy=none,users[1].accessKey=accessKey2,users[1].secretRef=existingSecret,users[1].secretKey=password,users[1].policy=none minio/minio +``` + +Description of the configuration parameters used above - + +- `users[].accessKey` - accessKey of user +- `users[].secretKey` - secretKey of usersecretRef +- `users[].existingSecret` - secret name that contains the secretKey of user +- `users[].existingSecretKey` - data key in existingSecret secret containing the secretKey +- `users[].policy` - name of the policy to assign to user + +### Create service account after install + +Install the chart, specifying the service accounts you want to create after install: + +```bash +helm install --set svcaccts[0].accessKey=accessKey,svcaccts[0].secretKey=secretKey,svcaccts[0].user=parentUser,svcaccts[1].accessKey=accessKey2,svcaccts[1].secretRef=existingSecret,svcaccts[1].secretKey=password,svcaccts[1].user=parentUser2 minio/minio +``` + +Description of the configuration parameters used above - + +- `svcaccts[].accessKey` - accessKey of service account +- `svcaccts[].secretKey` - secretKey of svcacctsecretRef +- `svcaccts[].existingSecret` - secret name that contains the secretKey of service account +- `svcaccts[].existingSecretKey` - data key in existingSecret secret containing the secretKey +- `svcaccts[].user` - name of the parent user to assign to service account + +## Uninstalling the Chart + +Assuming your release is named as `my-release`, delete it using the command: + +```bash +helm delete my-release +``` + +or + +```bash +helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. diff --git a/opencloud/charts/loki/charts/minio/templates/NOTES.txt b/opencloud/charts/loki/charts/minio/templates/NOTES.txt new file mode 100644 index 0000000..ba51b4c --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/NOTES.txt @@ -0,0 +1,43 @@ +{{- if eq .Values.service.type "ClusterIP" "NodePort" }} +MinIO can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: +{{ template "minio.fullname" . }}.{{ .Release.Namespace }}.{{ .Values.clusterDomain }} + +To access MinIO from localhost, run the below commands: + + 1. export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + + 2. kubectl port-forward $POD_NAME 9000 --namespace {{ .Release.Namespace }} + +Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ + +You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: + + 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart + + 2. export MC_HOST_{{ template "minio.fullname" . }}-local=http://$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:{{ .Values.service.port }} + + 3. mc ls {{ template "minio.fullname" . }}-local + +{{- end }} +{{- if eq .Values.service.type "LoadBalancer" }} +MinIO can be accessed via port {{ .Values.service.port }} on an external IP address. Get the service external IP address by: +kubectl get svc --namespace {{ .Release.Namespace }} -l app={{ template "minio.fullname" . }} + +Note that the public IP may take a couple of minutes to be available. + +You can now access MinIO server on http://:9000. Follow the below steps to connect to MinIO server with mc client: + + 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart + + 2. export MC_HOST_{{ template "minio.fullname" . }}-local=http://$(kubectl get secret {{ template "minio.secretName" . }} --namespace {{ .Release.Namespace }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@:{{ .Values.service.port }} + + 3. mc ls {{ template "minio.fullname" . }} + +Alternately, you can use your browser or the MinIO SDK to access the server - https://min.io/docs/minio/linux/reference/minio-server/minio-server.html +{{- end }} + +{{ if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} +Note: Since NetworkPolicy is enabled, only pods with label +{{ template "minio.fullname" . }}-client=true" +will be able to connect to this minio cluster. +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_create_bucket.txt b/opencloud/charts/loki/charts/minio/templates/_helper_create_bucket.txt new file mode 100644 index 0000000..83b8dcb --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_create_bucket.txt @@ -0,0 +1,122 @@ +#!/bin/sh +set -e # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 +} + +# checkBucketExists ($bucket) +# Check if the bucket exists, by using the exit code of `mc ls` +checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? +} + +# createBucket ($bucket, $policy, $purge) +# Ensure bucket exists, purging if asked to +createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.buckets }} +{{ $global := . }} +# Create the buckets +{{- range .Values.buckets }} +createBucket {{ tpl .name $global }} {{ .policy | default "none" | quote }} {{ .purge | default false }} {{ .versioning | default false }} {{ .objectlocking | default false }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_create_policy.txt b/opencloud/charts/loki/charts/minio/templates/_helper_create_policy.txt new file mode 100644 index 0000000..aa58495 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_create_policy.txt @@ -0,0 +1,75 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkPolicyExists ($policy) +# Check if the policy exists, by using the exit code of `mc admin policy info` +checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? +} + +# createPolicy($name, $filename) +createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.policies }} +# Create the policies +{{- range $idx, $policy := .Values.policies }} +createPolicy {{ $policy.name }} policy_{{ $idx }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_create_svcacct.txt b/opencloud/charts/loki/charts/minio/templates/_helper_create_svcacct.txt new file mode 100644 index 0000000..5c8aec4 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_create_svcacct.txt @@ -0,0 +1,106 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. +# Special characters for example : ',",<,>,{,} +MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkSvcacctExists () +# Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` +checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? +} + +# createSvcacct ($user) +createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.svcaccts }} +{{ $global := . }} +# Create the svcaccts +{{- range $idx, $svc := .Values.svcaccts }} +echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- if .existingSecret }} +cat /config/secrets-svc/{{ tpl .existingSecret $global }}/{{ tpl .existingSecretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +# Add a new line if it doesn't exist +echo >> $MINIO_ACCESSKEY_SECRETKEY_TMP +{{ else }} +echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- end }} +{{- if $svc.policy}} +createSvcacct {{ .user }} svc_policy_{{ $idx }} +{{ else }} +createSvcacct {{ .user }} +{{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_create_user.txt b/opencloud/charts/loki/charts/minio/templates/_helper_create_user.txt new file mode 100644 index 0000000..bfb79be --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_create_user.txt @@ -0,0 +1,107 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. +# Special characters for example : ',",<,>,{,} +MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkUserExists () +# Check if the user exists, by using the exit code of `mc admin user info` +checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? +} + +# createUser ($policy) +createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.users }} +{{ $global := . }} +# Create the users +{{- range .Values.users }} +echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- if .existingSecret }} +cat /config/secrets/{{ tpl .existingSecret $global }}/{{ tpl .existingSecretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +# Add a new line if it doesn't exist +echo >> $MINIO_ACCESSKEY_SECRETKEY_TMP +createUser {{ .policy }} +{{ else }} +echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +createUser {{ .policy }} +{{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_custom_command.txt b/opencloud/charts/loki/charts/minio/templates/_helper_custom_command.txt new file mode 100644 index 0000000..b583a77 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_custom_command.txt @@ -0,0 +1,58 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# runCommand ($@) +# Run custom mc command +runCommand() { + ${MC} "$@" + return $? +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.customCommands }} +# Run custom commands +{{- range .Values.customCommands }} +runCommand {{ .command }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/_helper_policy.tpl b/opencloud/charts/loki/charts/minio/templates/_helper_policy.tpl new file mode 100644 index 0000000..8be998e --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helper_policy.tpl @@ -0,0 +1,28 @@ +{{- $statements_length := len .statements -}} +{{- $statements_length := sub $statements_length 1 -}} +{ + "Version": "2012-10-17", + "Statement": [ +{{- range $i, $statement := .statements }} + { + "Effect": "{{ $statement.effect | default "Allow" }}", + "Action": [ +"{{ $statement.actions | join "\",\n\"" }}" + ]{{ if $statement.resources }}, + "Resource": [ +"{{ $statement.resources | join "\",\n\"" }}" + ]{{ end }} +{{- if $statement.conditions }} +{{- $condition_len := len $statement.conditions }} +{{- $condition_len := sub $condition_len 1 }} + , + "Condition": { + {{- range $k,$v := $statement.conditions }} + {{- range $operator,$object := $v }} + "{{ $operator }}": { {{ $object }} }{{- if lt $k $condition_len }},{{- end }} + {{- end }}{{- end }} + }{{- end }} + }{{ if lt $i $statements_length }},{{end }} +{{- end }} + ] +} diff --git a/opencloud/charts/loki/charts/minio/templates/_helpers.tpl b/opencloud/charts/loki/charts/minio/templates/_helpers.tpl new file mode 100644 index 0000000..1cb209e --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/_helpers.tpl @@ -0,0 +1,218 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minio.name" -}} + {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- $name := default .Chart.Name .Values.nameOverride -}} + {{- if contains $name .Release.Name -}} + {{- .Release.Name | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio.chart" -}} + {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "minio.networkPolicy.apiVersion" -}} + {{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.Version -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare ">=1.7-0, <1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else if semverCompare "^1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "minio.deployment.apiVersion" -}} + {{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} + {{- print "apps/v1beta2" -}} + {{- else -}} + {{- print "apps/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "minio.statefulset.apiVersion" -}} + {{- if semverCompare "<1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "apps/v1beta2" -}} + {{- else -}} + {{- print "apps/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "minio.ingress.apiVersion" -}} + {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for console ingress. +*/}} +{{- define "minio.consoleIngress.apiVersion" -}} + {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Determine secret name. +*/}} +{{- define "minio.secretName" -}} + {{- if .Values.existingSecret -}} + {{- .Values.existingSecret }} + {{- else -}} + {{- include "minio.fullname" . -}} + {{- end -}} +{{- end -}} + +{{/* +Determine name for scc role and rolebinding +*/}} +{{- define "minio.sccRoleName" -}} + {{- printf "%s-%s" "scc" (include "minio.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Properly format optional additional arguments to MinIO binary +*/}} +{{- define "minio.extraArgs" -}} +{{- range .Values.extraArgs -}} +{{ " " }}{{ . }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "minio.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- end -}} + +{{/* +Formats volumeMount for MinIO TLS keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolumeMount" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + mountPath: {{ .Values.certsPath }} +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $casPath := printf "%s/CAs" .Values.certsPath | clean }} +- name: trusted-cert-secret-volume + mountPath: {{ $casPath }} +{{- end }} +{{- end -}} + +{{/* +Formats volume for MinIO TLS keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolume" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + secret: + secretName: {{ tpl .Values.tls.certSecret $ }} + items: + - key: {{ .Values.tls.publicCrt }} + path: public.crt + - key: {{ .Values.tls.privateKey }} + path: private.key +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $certSecret := eq .Values.trustedCertsSecret "" | ternary .Values.tls.certSecret .Values.trustedCertsSecret }} +{{- $publicCrt := eq .Values.trustedCertsSecret "" | ternary .Values.tls.publicCrt "" }} +- name: trusted-cert-secret-volume + secret: + secretName: {{ $certSecret }} + {{- if ne $publicCrt "" }} + items: + - key: {{ $publicCrt }} + path: public.crt + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Returns the available value for certain key in an existing secret (if it exists), +otherwise it generates a random value. +*/}} +{{- define "minio.getValueFromSecret" }} + {{- $len := (default 16 .Length) | int -}} + {{- $obj := (lookup "v1" "Secret" .Namespace .Name).data -}} + {{- if $obj }} + {{- index $obj .Key | b64dec -}} + {{- else -}} + {{- randAlphaNum $len -}} + {{- end -}} +{{- end }} + +{{- define "minio.root.username" -}} + {{- if .Values.rootUser }} + {{- .Values.rootUser | toString }} + {{- else }} + {{- include "minio.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (include "minio.fullname" .) "Length" 20 "Key" "rootUser") }} + {{- end }} +{{- end -}} + +{{- define "minio.root.password" -}} + {{- if .Values.rootPassword }} + {{- .Values.rootPassword | toString }} + {{- else }} + {{- include "minio.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (include "minio.fullname" .) "Length" 40 "Key" "rootPassword") }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/charts/minio/templates/ciliumnetworkpolicy.yaml b/opencloud/charts/loki/charts/minio/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000..1dc91bc --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "cilium") }} +kind: CiliumNetworkPolicy +apiVersion: cilium.io/v2 +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + endpointSelector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + ingress: + - toPorts: + - ports: + - port: "{{ .Values.minioAPIPort }}" + protocol: TCP + - port: "{{ .Values.minioConsolePort }}" + protocol: TCP + {{- if not .Values.networkPolicy.allowExternal }} + fromEndpoints: + - matchLabels: + {{ template "minio.name" . }}-client: "true" + {{- end }} + egress: + {{- range $entity := .Values.networkPolicy.egressEntities }} + - toEntities: + - {{ $entity }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/configmap.yaml b/opencloud/charts/loki/charts/minio/templates/configmap.yaml new file mode 100644 index 0000000..47f64cc --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + initialize: |- + {{- include (print $.Template.BasePath "/_helper_create_bucket.txt") . | nindent 4 }} + add-user: |- + {{- include (print $.Template.BasePath "/_helper_create_user.txt") . | nindent 4 }} + add-policy: |- + {{- include (print $.Template.BasePath "/_helper_create_policy.txt") . | nindent 4 }} + {{- range $idx, $policy := .Values.policies }} + # Policy: {{ $policy.name }} + policy_{{ $idx }}.json: |- + {{- include (print $.Template.BasePath "/_helper_policy.tpl") . | nindent 4 }} + {{ end }} + {{- range $idx, $svc := .Values.svcaccts }} + {{- if $svc.policy }} + # SVC: {{ $svc.accessKey }} + svc_policy_{{ $idx }}.json: |- + {{- include (print $.Template.BasePath "/_helper_policy.tpl") .policy | nindent 4 }} + {{- end }} + {{- end }} + add-svcacct: |- + {{- include (print $.Template.BasePath "/_helper_create_svcacct.txt") . | nindent 4 }} + custom-command: |- + {{- include (print $.Template.BasePath "/_helper_custom_command.txt") . | nindent 4 }} diff --git a/opencloud/charts/loki/charts/minio/templates/console-ingress.yaml b/opencloud/charts/loki/charts/minio/templates/console-ingress.yaml new file mode 100644 index 0000000..79a2b1b --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/console-ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.consoleIngress.enabled -}} +{{- $fullName := printf "%s-console" (include "minio.fullname" .) -}} +{{- $servicePort := .Values.consoleService.port -}} +{{- $ingressPath := .Values.consoleIngress.path -}} +apiVersion: {{ template "minio.consoleIngress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.consoleIngress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.consoleIngress.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.consoleIngress.ingressClassName }} + ingressClassName: {{ .Values.consoleIngress.ingressClassName }} + {{- end }} + {{- if .Values.consoleIngress.tls }} + tls: + {{- range .Values.consoleIngress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.consoleIngress.hosts }} + - http: + paths: + - path: {{ $ingressPath }} + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if . }} + host: {{ tpl . $ | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/console-service.yaml b/opencloud/charts/loki/charts/minio/templates/console-service.yaml new file mode 100644 index 0000000..f09e3f3 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/console-service.yaml @@ -0,0 +1,45 @@ +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }}-console + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.consoleService.annotations }} + annotations: {{- toYaml .Values.consoleService.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.consoleService.type }} + {{- if and (eq .Values.consoleService.type "ClusterIP") .Values.consoleService.clusterIP }} + clusterIP: {{ .Values.consoleService.clusterIP }} + {{- end }} + {{- if or (eq .Values.consoleService.type "LoadBalancer") (eq .Values.consoleService.type "NodePort") }} + externalTrafficPolicy: {{ .Values.consoleService.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.consoleService.type "LoadBalancer") .Values.consoleService.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ .Values.consoleService.loadBalancerSourceRanges }} + {{ end }} + {{- if and (eq .Values.consoleService.type "LoadBalancer") (not (empty .Values.consoleService.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.consoleService.loadBalancerIP }} + {{- end }} + ports: + - name: {{ $scheme }} + port: {{ .Values.consoleService.port }} + protocol: TCP + {{- if (and (eq .Values.consoleService.type "NodePort") ( .Values.consoleService.nodePort)) }} + nodePort: {{ .Values.consoleService.nodePort }} + {{- else }} + targetPort: {{ .Values.minioConsolePort }} + {{- end }} + {{- if .Values.consoleService.externalIPs }} + externalIPs: + {{- range $i , $ip := .Values.consoleService.externalIPs }} + - {{ $ip }} + {{- end }} + {{- end }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} diff --git a/opencloud/charts/loki/charts/minio/templates/deployment.yaml b/opencloud/charts/loki/charts/minio/templates/deployment.yaml new file mode 100644 index 0000000..4c57010 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/deployment.yaml @@ -0,0 +1,213 @@ +{{- if eq .Values.mode "standalone" }} +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +apiVersion: {{ template "minio.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.additionalLabels }} + {{- toYaml .Values.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.additionalAnnotations }} + annotations: {{- toYaml .Values.additionalAnnotations | nindent 4 }} + {{- end }} +spec: + strategy: + type: {{ .Values.deploymentUpdate.type }} + {{- if eq .Values.deploymentUpdate.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ .Values.deploymentUpdate.maxSurge }} + maxUnavailable: {{ .Values.deploymentUpdate.maxUnavailable }} + {{- end }} + replicas: 1 + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + annotations: + {{- if not .Values.ignoreChartChecksums }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | trimSuffix "\n" | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.runtimeClassName }} + runtimeClassName: "{{ .Values.runtimeClassName }}" + {{- end }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + {{ omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{ if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio server {{ $bucketRoot }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template "minio.extraArgs" . }}" + volumeMounts: + - name: minio-user + mountPath: "/tmp/credentials" + readOnly: true + - name: export + mountPath: {{ .Values.mountPath }} + {{- if and .Values.persistence.enabled .Values.persistence.subPath }} + subPath: "{{ .Values.persistence.subPath }}" + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + mountPath: "/tmp/minio-config-env" + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: {{ $scheme }} + containerPort: {{ .Values.minioAPIPort }} + - name: {{ $scheme }}-console + containerPort: {{ .Values.minioConsolePort }} + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootPassword + {{- if .Values.extraSecret }} + - name: MINIO_CONFIG_ENV_FILE + value: "/tmp/minio-config-env/config.env" + {{- end }} + {{- if .Values.metrics.serviceMonitor.public }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + {{- end }} + {{- if .Values.oidc.enabled }} + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: {{ .Values.oidc.configUrl }} + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientIdKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientIdKey }} + {{- else }} + value: {{ .Values.oidc.clientId }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientSecretKey }} + {{- else }} + value: {{ .Values.oidc.clientSecret }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: {{ .Values.oidc.claimName }} + - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX + value: {{ .Values.oidc.claimPrefix }} + - name: MINIO_IDENTITY_OPENID_SCOPES + value: {{ .Values.oidc.scopes }} + - name: MINIO_IDENTITY_OPENID_COMMENT + value: {{ .Values.oidc.comment }} + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI + value: {{ .Values.oidc.redirectUri }} + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: {{ .Values.oidc.displayName }} + {{- end }} + {{- if .Values.etcd.endpoints }} + - name: MINIO_ETCD_ENDPOINTS + value: {{ join "," .Values.etcd.endpoints | quote }} + {{- if .Values.etcd.clientCert }} + - name: MINIO_ETCD_CLIENT_CERT + value: "/tmp/credentials/etcd_client_cert.pem" + {{- end }} + {{- if .Values.etcd.clientCertKey }} + - name: MINIO_ETCD_CLIENT_CERT_KEY + value: "/tmp/credentials/etcd_client_cert_key.pem" + {{- end }} + {{- if .Values.etcd.pathPrefix }} + - name: MINIO_ETCD_PATH_PREFIX + value: {{ .Values.etcd.pathPrefix }} + {{- end }} + {{- if .Values.etcd.corednsPathPrefix }} + - name: MINIO_ETCD_COREDNS_PATH + value: {{ .Values.etcd.corednsPathPrefix }} + {{- end }} + {{- end }} + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ tpl $val $ | quote }} + {{- end }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12}} + {{- end }} + {{- end }} + {{- with .Values.extraContainers }} + {{- if eq (typeOf .) "string" }} + {{- tpl . $ | nindent 8 }} + {{- else }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: export + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "minio.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + secret: + secretName: {{ .Values.extraSecret }} + {{- end }} + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/ingress.yaml b/opencloud/charts/loki/charts/minio/templates/ingress.yaml new file mode 100644 index 0000000..1a564c6 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "minio.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: {{ template "minio.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - http: + paths: + - path: {{ $ingressPath }} + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if . }} + host: {{ tpl . $ | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/networkpolicy.yaml b/opencloud/charts/loki/charts/minio/templates/networkpolicy.yaml new file mode 100644 index 0000000..b9c0771 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/networkpolicy.yaml @@ -0,0 +1,26 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: {{ template "minio.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + ingress: + - ports: + - port: {{ .Values.minioAPIPort }} + - port: {{ .Values.minioConsolePort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "minio.name" . }}-client: "true" + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/poddisruptionbudget.yaml b/opencloud/charts/loki/charts/minio/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..a5f90a0 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/poddisruptionbudget.yaml @@ -0,0 +1,17 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodDisruptionBudget" }} +apiVersion: policy/v1beta1 +{{- else }} +apiVersion: policy/v1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: minio + labels: + app: {{ template "minio.name" . }} +spec: + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + app: {{ template "minio.name" . }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/post-job.yaml b/opencloud/charts/loki/charts/minio/templates/post-job.yaml new file mode 100644 index 0000000..955d655 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/post-job.yaml @@ -0,0 +1,258 @@ +{{- if or .Values.buckets .Values.users .Values.policies .Values.customCommands .Values.svcaccts }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "minio.fullname" . }}-post-job + labels: + app: {{ template "minio.name" . }}-post-job + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + {{- with .Values.postJob.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + metadata: + labels: + app: {{ template "minio.name" . }}-job + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.postJob.podAnnotations }} + annotations: {{- toYaml .Values.postJob.podAnnotations | nindent 8 }} + {{- end }} + spec: + restartPolicy: OnFailure + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- toYaml .Values.postJob.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.postJob.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postJob.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.postJob.securityContext.enabled }} + securityContext: {{ omit .Values.postJob.securityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: {{ template "minio.fullname" . }} + - secret: + name: {{ template "minio.secretName" . }} + {{- range (concat .Values.users (default (list) .Values.svcaccts)) }} + {{- if .existingSecret }} + - secret: + name: {{ tpl .existingSecret $ }} + items: + - key: {{ .existingSecretKey }} + path: secrets/{{ tpl .existingSecret $ }}/{{ tpl .existingSecretKey $ }} + {{- end }} + {{- end }} + {{- range ( default list .Values.svcaccts ) }} + {{- if .existingSecret }} + - secret: + name: {{ tpl .existingSecret $ }} + items: + - key: {{ .existingSecretKey }} + path: secrets-svc/{{ tpl .existingSecret $ }}/{{ tpl .existingSecretKey $ }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + secret: + secretName: {{ .Values.tls.certSecret }} + items: + - key: {{ .Values.tls.publicCrt }} + path: CAs/public.crt + {{- end }} + {{- if .Values.customCommandJob.extraVolumes }} + {{- toYaml .Values.customCommandJob.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + {{- if .Values.policies }} + initContainers: + - name: minio-make-policy + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makePolicyJob.securityContext.enabled }} + {{- with .Values.makePolicyJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makePolicyJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/add-policy; EV=$?; {{ .Values.makePolicyJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/add-policy" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makePolicyJob.resources | nindent 12 }} + {{- end }} + containers: + {{- if .Values.buckets }} + - name: minio-make-bucket + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeBucketJob.securityContext.enabled }} + {{- with .Values.makeBucketJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeBucketJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/initialize; EV=$?; {{ .Values.makeBucketJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/initialize" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeBucketJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.users }} + - name: minio-make-user + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeUserJob.securityContext.enabled }} + {{- with .Values.makeUserJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeUserJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/add-user; EV=$?; {{ .Values.makeUserJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/add-user" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeUserJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.customCommands }} + - name: minio-custom-command + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.customCommandJob.securityContext.enabled }} + {{- with .Values.customCommandJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.customCommandJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/custom-command; EV=$?; {{ .Values.customCommandJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/custom-command" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + {{- if .Values.customCommandJob.extraVolumeMounts }} + {{- toYaml .Values.customCommandJob.extraVolumeMounts | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.customCommandJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.svcaccts }} + - name: minio-make-svcacct + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeServiceAccountJob.securityContext.enabled }} + {{- with .Values.makeServiceAccountJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeServiceAccountJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: ["/bin/sh /config/add-svcacct; EV=$?; {{ .Values.makeServiceAccountJob.exitCommand }} && exit $EV" ] + {{- else }} + command: ["/bin/sh", "/config/add-svcacct"] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeServiceAccountJob.resources | nindent 12 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/pvc.yaml b/opencloud/charts/loki/charts/minio/templates/pvc.yaml new file mode 100644 index 0000000..60f5267 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/pvc.yaml @@ -0,0 +1,32 @@ +{{- if eq .Values.mode "standalone" }} +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.persistence.annotations }} + annotations: {{- toYaml .Values.persistence.annotations | nindent 4 }} + {{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" + {{- end }} + {{- end }} + {{- if .Values.persistence.volumeName }} + volumeName: "{{ .Values.persistence.volumeName }}" + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/secrets.yaml b/opencloud/charts/loki/charts/minio/templates/secrets.yaml new file mode 100644 index 0000000..476c3da --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/secrets.yaml @@ -0,0 +1,21 @@ +{{- if not .Values.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "minio.secretName" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + rootUser: {{ include "minio.root.username" . | b64enc | quote }} + rootPassword: {{ include "minio.root.password" . | b64enc | quote }} + {{- if .Values.etcd.clientCert }} + etcd_client.crt: {{ .Values.etcd.clientCert | toString | b64enc | quote }} + {{- end }} + {{- if .Values.etcd.clientCertKey }} + etcd_client.key: {{ .Values.etcd.clientCertKey | toString | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/securitycontextconstraints.yaml b/opencloud/charts/loki/charts/minio/templates/securitycontextconstraints.yaml new file mode 100644 index 0000000..4bac7e3 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/securitycontextconstraints.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.securityContext.enabled .Values.persistence.enabled (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: [] +readOnlyRootFilesystem: false +defaultAddCapabilities: [] +requiredDropCapabilities: +- KILL +- MKNOD +- SETUID +- SETGID +fsGroup: + type: MustRunAs + ranges: + - max: {{ .Values.securityContext.fsGroup }} + min: {{ .Values.securityContext.fsGroup }} +runAsUser: + type: MustRunAs + uid: {{ .Values.securityContext.runAsUser }} +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/service.yaml b/opencloud/charts/loki/charts/minio/templates/service.yaml new file mode 100644 index 0000000..d872cd0 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/service.yaml @@ -0,0 +1,46 @@ +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + monitoring: "true" + {{- if .Values.service.annotations }} + annotations: {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "ClusterIP") .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} + {{ end }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }} + loadBalancerIP: {{ default "" .Values.service.loadBalancerIP | quote }} + {{- end }} + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP + {{- if (and (eq .Values.service.type "NodePort") ( .Values.service.nodePort)) }} + nodePort: {{ .Values.service.nodePort }} + {{- else }} + targetPort: {{ .Values.minioAPIPort }} + {{- end }} + {{- if .Values.service.externalIPs }} + externalIPs: + {{- range $i , $ip := .Values.service.externalIPs }} + - {{ $ip }} + {{- end }} + {{- end }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} diff --git a/opencloud/charts/loki/charts/minio/templates/serviceaccount.yaml b/opencloud/charts/loki/charts/minio/templates/serviceaccount.yaml new file mode 100644 index 0000000..0784015 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | quote }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/servicemonitor.yaml b/opencloud/charts/loki/charts/minio/templates/servicemonitor.yaml new file mode 100644 index 0000000..f875a85 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/servicemonitor.yaml @@ -0,0 +1,112 @@ +{{- if and .Values.metrics.serviceMonitor.enabled .Values.metrics.serviceMonitor.includeNode }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "minio.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.metrics.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + endpoints: + {{- if .Values.tls.enabled }} + - port: https + scheme: https + tlsConfig: + ca: + secret: + name: {{ .Values.tls.certSecret }} + key: {{ .Values.tls.publicCrt }} + serverName: {{ template "minio.fullname" . }} + {{- else }} + - port: http + scheme: http + {{- end }} + path: /minio/v2/metrics/node + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelConfigs }} + {{- toYaml .Values.metrics.serviceMonitor.relabelConfigs | nindent 6 }} + {{- end }} + {{- if not .Values.metrics.serviceMonitor.public }} + bearerTokenSecret: + name: {{ template "minio.fullname" . }}-prometheus + key: token + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + selector: + matchLabels: + app: {{ include "minio.name" . }} + release: {{ .Release.Name }} + monitoring: "true" +{{- end }} +{{- if .Values.metrics.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ template "minio.fullname" . }}-cluster + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + jobName: {{ template "minio.fullname" . }} + {{- if .Values.tls.enabled }} + tlsConfig: + ca: + secret: + name: {{ .Values.tls.certSecret }} + key: {{ .Values.tls.publicCrt }} + serverName: {{ template "minio.fullname" . }} + {{- end }} + prober: + url: {{ template "minio.fullname" . }}.{{ .Release.Namespace }}:{{ .Values.service.port }} + path: /minio/v2/metrics/cluster + {{- if .Values.tls.enabled }} + scheme: https + {{- else }} + scheme: http + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelConfigsCluster }} + {{- toYaml .Values.metrics.serviceMonitor.relabelConfigsCluster | nindent 2 }} + {{- end }} + targets: + staticConfig: + static: + - {{ template "minio.fullname" . }}.{{ .Release.Namespace }} + {{- if not .Values.metrics.serviceMonitor.public }} + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + bearerTokenSecret: + name: {{ template "minio.fullname" . }}-prometheus + key: token + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/templates/statefulset.yaml b/opencloud/charts/loki/charts/minio/templates/statefulset.yaml new file mode 100644 index 0000000..d671eaa --- /dev/null +++ b/opencloud/charts/loki/charts/minio/templates/statefulset.yaml @@ -0,0 +1,267 @@ +{{- if eq .Values.mode "distributed" }} +{{ $poolCount := .Values.pools | int }} +{{ $nodeCount := .Values.replicas | int }} +{{ $replicas := mul $poolCount $nodeCount }} +{{ $drivesPerNode := .Values.drivesPerNode | int }} +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +{{ $mountPath := .Values.mountPath }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +{{ $subPath := .Values.persistence.subPath }} +{{ $penabled := .Values.persistence.enabled }} +{{ $accessMode := .Values.persistence.accessMode }} +{{ $storageClass := .Values.persistence.storageClass }} +{{ $psize := .Values.persistence.size }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }}-svc + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + publishNotReadyAddresses: true + clusterIP: None + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.minioAPIPort }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} +--- +apiVersion: {{ template "minio.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.additionalLabels }} + {{- toYaml .Values.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.additionalAnnotations }} + annotations: {{- toYaml .Values.additionalAnnotations | nindent 4 }} + {{- end }} +spec: + updateStrategy: + type: {{ .Values.statefulSetUpdate.updateStrategy }} + podManagementPolicy: "Parallel" + serviceName: {{ template "minio.fullname" . }}-svc + replicas: {{ $replicas }} + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + annotations: + {{- if not .Values.ignoreChartChecksums }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.runtimeClassName }} + runtimeClassName: "{{ .Values.runtimeClassName }}" + {{- end }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ + "/bin/sh", + "-ce", + "/usr/bin/docker-entrypoint.sh minio server {{- range $i := until $poolCount }}{{ $factor := mul $i $nodeCount }}{{ $endIndex := add $factor $nodeCount }}{{ $beginIndex := mul $i $nodeCount }} {{ $scheme }}://{{ template `minio.fullname` $ }}-{{ `{` }}{{ $beginIndex }}...{{ sub $endIndex 1 }}{{ `}`}}.{{ template `minio.fullname` $ }}-svc.{{ $.Release.Namespace }}.svc{{if (gt $drivesPerNode 1)}}{{ $bucketRoot }}-{{ `{` }}0...{{ sub $drivesPerNode 1 }}{{ `}` }}{{ else }}{{ $bucketRoot }}{{end }}{{- end }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template `minio.extraArgs` . }}" + ] + volumeMounts: + {{- if $penabled }} + {{- if (gt $drivesPerNode 1) }} + {{- range $i := until $drivesPerNode }} + - name: export-{{ $i }} + mountPath: {{ $mountPath }}-{{ $i }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- else }} + - name: export + mountPath: {{ $mountPath }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + mountPath: "/tmp/minio-config-env" + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: {{ $scheme }} + containerPort: {{ .Values.minioAPIPort }} + - name: {{ $scheme }}-console + containerPort: {{ .Values.minioConsolePort }} + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootPassword + {{- if .Values.extraSecret }} + - name: MINIO_CONFIG_ENV_FILE + value: "/tmp/minio-config-env/config.env" + {{- end }} + {{- if .Values.metrics.serviceMonitor.public }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + {{- end }} + {{- if .Values.oidc.enabled }} + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: {{ .Values.oidc.configUrl }} + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientIdKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientIdKey }} + {{- else }} + value: {{ .Values.oidc.clientId }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientSecretKey }} + {{- else }} + value: {{ .Values.oidc.clientSecret }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: {{ .Values.oidc.claimName }} + - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX + value: {{ .Values.oidc.claimPrefix }} + - name: MINIO_IDENTITY_OPENID_SCOPES + value: {{ .Values.oidc.scopes }} + - name: MINIO_IDENTITY_OPENID_COMMENT + value: {{ .Values.oidc.comment }} + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI + value: {{ .Values.oidc.redirectUri }} + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: {{ .Values.oidc.displayName }} + {{- end }} + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ tpl $val $ | quote }} + {{- end }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12}} + {{- end }} + {{- end }} + {{- with .Values.extraContainers }} + {{- if eq (typeOf .) "string" }} + {{- tpl . $ | nindent 8 }} + {{- else }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and (gt $replicas 1) (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor "19") }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + volumes: + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- if .Values.extraSecret }} + - name: extra-secret + secret: + secretName: {{ .Values.extraSecret }} + {{- end }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + {{- if gt $drivesPerNode 1 }} + {{- range $diskId := until $drivesPerNode}} + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: export-{{ $diskId }} + {{- if $.Values.persistence.annotations }} + annotations: {{- toYaml $.Values.persistence.annotations | nindent 10 }} + {{- end }} + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} + {{- else }} + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: export + {{- if $.Values.persistence.annotations }} + annotations: {{- toYaml $.Values.persistence.annotations | nindent 10 }} + {{- end }} + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/minio/values.yaml b/opencloud/charts/loki/charts/minio/values.yaml new file mode 100644 index 0000000..8eb9863 --- /dev/null +++ b/opencloud/charts/loki/charts/minio/values.yaml @@ -0,0 +1,593 @@ +## Provide a name in place of minio for `app:` labels +## +nameOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## set kubernetes cluster domain where minio is running +## +clusterDomain: cluster.local + +## Set default image, imageTag, and imagePullPolicy. mode is used to indicate the +## +image: + repository: quay.io/minio/minio + tag: RELEASE.2024-04-18T19-09-19Z + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +## Set default image, imageTag, and imagePullPolicy for the `mc` (the minio +## client used to create a default bucket). +## +mcImage: + repository: quay.io/minio/mc + tag: RELEASE.2024-04-18T16-45-29Z + pullPolicy: IfNotPresent + +## minio mode, i.e. standalone or distributed +mode: distributed ## other supported values are "standalone" + +## Additional labels to include with deployment or statefulset +additionalLabels: {} + +## Additional annotations to include with deployment or statefulset +additionalAnnotations: {} + +## Typically the deployment/statefulset includes checksums of secrets/config, +## So that when these change on a subsequent helm install, the deployment/statefulset +## is restarted. This can result in unnecessary restarts under GitOps tooling such as +## flux, so set to "true" to disable this behaviour. +ignoreChartChecksums: false + +## Additional arguments to pass to minio binary +extraArgs: [] +# example for enabling FTP: +# - --ftp=\"address=:8021\" +# - --ftp=\"passive-port-range=10000-10010\" + +## Additional volumes to minio container +extraVolumes: [] + +## Additional volumeMounts to minio container +extraVolumeMounts: [] + +## Additional sidecar containers +extraContainers: [] + +## Internal port number for MinIO S3 API container +## Change service.port to change external port number +minioAPIPort: "9000" + +## Internal port number for MinIO Browser Console container +## Change consoleService.port to change external port number +minioConsolePort: "9001" + +## Update strategy for Deployments +deploymentUpdate: + type: RollingUpdate + maxUnavailable: 0 + maxSurge: 100% + +## Update strategy for StatefulSets +statefulSetUpdate: + updateStrategy: RollingUpdate + +## Pod priority settings +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" + +## Pod runtime class name +## ref https://kubernetes.io/docs/concepts/containers/runtime-class/ +## +runtimeClassName: "" + +## Set default rootUser, rootPassword +## rootUser and rootPassword is generated when not set +## Distributed MinIO ref: https://min.io/docs/minio/linux/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html +## +rootUser: "" +rootPassword: "" + +## Use existing Secret that store following variables: +## +## | Chart var | .data. in Secret | +## |:----------------------|:-------------------------| +## | rootUser | rootUser | +## | rootPassword | rootPassword | +## +## All mentioned variables will be ignored in values file. +## .data.rootUser and .data.rootPassword are mandatory, +## others depend on enabled status of corresponding sections. +existingSecret: "" + +## Directory on the MinIO pof +certsPath: "/etc/minio/certs/" +configPathmc: "/etc/minio/mc/" + +## Path where PV would be mounted on the MinIO Pod +mountPath: "/export" +## Override the root directory which the minio server should serve from. +## If left empty, it defaults to the value of {{ .Values.mountPath }} +## If defined, it must be a sub-directory of the path specified in {{ .Values.mountPath }} +## +bucketRoot: "" + +# Number of drives attached to a node +drivesPerNode: 1 +# Number of MinIO containers running +replicas: 16 +# Number of expanded MinIO clusters +pools: 1 + +## TLS Settings for MinIO +tls: + enabled: false + ## Create a secret with private.key and public.crt files and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + certSecret: "" + publicCrt: public.crt + privateKey: private.key + +## Trusted Certificates Settings for MinIO. Ref: https://min.io/docs/minio/linux/operations/network-encryption.html#third-party-certificate-authorities +## Bundle multiple trusted certificates into one secret and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret +## When using self-signed certificates, remember to include MinIO's own certificate in the bundle with key public.crt. +## If certSecret is left empty and tls is enabled, this chart installs the public certificate from .Values.tls.certSecret. +trustedCertsSecret: "" + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + enabled: true + annotations: {} + + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + existingClaim: "" + + ## minio data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + ## Storage class of PV to bind. By default it looks for standard storage class. + ## If the PV uses a different storage class, specify that here. + storageClass: "" + volumeName: "" + accessMode: ReadWriteOnce + size: 500Gi + + ## If subPath is set mount a sub folder of a volume instead of the root of the volume. + ## This is especially handy for volume plugins that don't natively support sub mounting (like glusterfs). + ## + subPath: "" + +## Expose the MinIO service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + type: ClusterIP + clusterIP: ~ + port: "9000" + nodePort: 32000 + loadBalancerIP: ~ + externalIPs: [] + annotations: {} + + ## service.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + #loadBalancerSourceRanges: + # - 10.10.10.0/24 + loadBalancerSourceRanges: [] + + ## service.externalTrafficPolicy minio service external traffic policy + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + +## Configure Ingress based on the documentation here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +## + +ingress: + enabled: false + ingressClassName: ~ + labels: {} + # node-role.kubernetes.io/ingress: platform + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.allow-http: "false" + # kubernetes.io/ingress.global-static-ip-name: "" + # nginx.ingress.kubernetes.io/secure-backends: "true" + # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0 + path: / + hosts: + - minio-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +consoleService: + type: ClusterIP + clusterIP: ~ + port: "9001" + nodePort: 32001 + loadBalancerIP: ~ + externalIPs: [] + annotations: {} + ## consoleService.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + #loadBalancerSourceRanges: + # - 10.10.10.0/24 + loadBalancerSourceRanges: [] + + ## servconsoleServiceice.externalTrafficPolicy minio service external traffic policy + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + +consoleIngress: + enabled: false + ingressClassName: ~ + labels: {} + # node-role.kubernetes.io/ingress: platform + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.allow-http: "false" + # kubernetes.io/ingress.global-static-ip-name: "" + # nginx.ingress.kubernetes.io/secure-backends: "true" + # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0 + path: / + hosts: + - console.minio-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] + +## Add stateful containers to have security context, if enabled MinIO will run as this +## user and group NOTE: securityContext is only enabled if persistence.enabled=true +securityContext: + enabled: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + +containerSecurityContext: + readOnlyRootFilesystem: false + +# Additational pod annotations +podAnnotations: {} + +# Additional pod labels +podLabels: {} + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 16Gi + +## List of policies to be created after minio install +## +## In addition to default policies [readonly|readwrite|writeonly|consoleAdmin|diagnostics] +## you can define additional policies with custom supported actions and resources +policies: [] +## writeexamplepolicy policy grants creation or deletion of buckets with name +## starting with example. In addition, grants objects write permissions on buckets starting with +## example. +# - name: writeexamplepolicy +# statements: +# - effect: Allow # this is the default +# resources: +# - 'arn:aws:s3:::example*/*' +# actions: +# - "s3:AbortMultipartUpload" +# - "s3:GetObject" +# - "s3:DeleteObject" +# - "s3:PutObject" +# - "s3:ListMultipartUploadParts" +# - resources: +# - 'arn:aws:s3:::example*' +# actions: +# - "s3:CreateBucket" +# - "s3:DeleteBucket" +# - "s3:GetBucketLocation" +# - "s3:ListBucket" +# - "s3:ListBucketMultipartUploads" +## readonlyexamplepolicy policy grants access to buckets with name starting with example. +## In addition, grants objects read permissions on buckets starting with example. +# - name: readonlyexamplepolicy +# statements: +# - resources: +# - 'arn:aws:s3:::example*/*' +# actions: +# - "s3:GetObject" +# - resources: +# - 'arn:aws:s3:::example*' +# actions: +# - "s3:GetBucketLocation" +# - "s3:ListBucket" +# - "s3:ListBucketMultipartUploads" +## conditionsexample policy creates all access to example bucket with aws:username="johndoe" and source ip range 10.0.0.0/8 and 192.168.0.0/24 only +# - name: conditionsexample +# statements: +# - resources: +# - 'arn:aws:s3:::example/*' +# actions: +# - 's3:*' +# conditions: +# - StringEquals: '"aws:username": "johndoe"' +# - IpAddress: | +# "aws:SourceIp": [ +# "10.0.0.0/8", +# "192.168.0.0/24" +# ] +# +## Additional Annotations for the Kubernetes Job makePolicyJob +makePolicyJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of users to be created after minio install +## +users: + ## Username, password and policy to be assigned to the user + ## Default policies are [readonly|readwrite|writeonly|consoleAdmin|diagnostics] + ## Add new policies as explained here https://min.io/docs/minio/kubernetes/upstream/administration/identity-access-management.html#access-management + ## NOTE: this will fail if LDAP is enabled in your MinIO deployment + ## make sure to disable this if you are using LDAP. + - accessKey: console + secretKey: console123 + policy: consoleAdmin + # Or you can refer to specific secret + #- accessKey: externalSecret + # existingSecret: my-secret + # existingSecretKey: password + # policy: readonly + +## Additional Annotations for the Kubernetes Job makeUserJob +makeUserJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of service accounts to be created after minio install +## +svcaccts: [] + ## accessKey, secretKey and parent user to be assigned to the service accounts + ## Add new service accounts as explained here https://min.io/docs/minio/kubernetes/upstream/administration/identity-access-management/minio-user-management.html#service-accounts + # - accessKey: console-svcacct + # secretKey: console123 + # user: console + ## Or you can refer to specific secret + # - accessKey: externalSecret + # existingSecret: my-secret + # existingSecretKey: password + # user: console + ## You also can pass custom policy + # - accessKey: console-svcacct + # secretKey: console123 + # user: console + # policy: + # statements: + # - resources: + # - 'arn:aws:s3:::example*/*' + # actions: + # - "s3:AbortMultipartUpload" + # - "s3:GetObject" + # - "s3:DeleteObject" + # - "s3:PutObject" + # - "s3:ListMultipartUploadParts" + +makeServiceAccountJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of buckets to be created after minio install +## +buckets: [] + # # Name of the bucket + # - name: bucket1 + # # Policy to be set on the + # # bucket [none|download|upload|public] + # policy: none + # # Purge if bucket exists already + # purge: false + # # set versioning for + # # bucket [true|false] + # versioning: false # remove this key if you do not want versioning feature + # # set objectlocking for + # # bucket [true|false] NOTE: versioning is enabled by default if you use locking + # objectlocking: false + # - name: bucket2 + # policy: none + # purge: false + # versioning: true + # # set objectlocking for + # # bucket [true|false] NOTE: versioning is enabled by default if you use locking + # objectlocking: false + +## Additional Annotations for the Kubernetes Job makeBucketJob +makeBucketJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of command to run after minio install +## NOTE: the mc command TARGET is always "myminio" +customCommands: + # - command: "admin policy attach myminio consoleAdmin --group='cn=ops,cn=groups,dc=example,dc=com'" + +## Additional Annotations for the Kubernetes Job customCommandJob +customCommandJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + ## Additional volumes to add to the post-job. + extraVolumes: [] + # - name: extra-policies + # configMap: + # name: my-extra-policies-cm + ## Additional volumeMounts to add to the custom commands container when + ## running the post-job. + extraVolumeMounts: [] + # - name: extra-policies + # mountPath: /mnt/extras/ + # Command to run after the main command on exit + exitCommand: "" + +## Merge jobs +postJob: + podAnnotations: {} + annotations: {} + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + nodeSelector: {} + tolerations: [] + affinity: {} + +## Use this field to add environment variables relevant to MinIO server. These fields will be passed on to MinIO container(s) +## when Chart is deployed +environment: + ## Please refer for comprehensive list https://min.io/docs/minio/linux/reference/minio-server/minio-server.html + ## MINIO_SUBNET_LICENSE: "License key obtained from https://subnet.min.io" + ## MINIO_BROWSER: "off" + +## The name of a secret in the same kubernetes namespace which contain secret values +## This can be useful for LDAP password, etc +## The key in the secret must be 'config.env' +## +extraSecret: ~ + +## OpenID Identity Management +## The following section documents environment variables for enabling external identity management using an OpenID Connect (OIDC)-compatible provider. +## See https://min.io/docs/minio/linux/operations/external-iam/configure-openid-external-identity-management.html for a tutorial on using these variables. +oidc: + enabled: false + configUrl: "https://identity-provider-url/.well-known/openid-configuration" + clientId: "minio" + clientSecret: "" + # Provide existing client secret from the Kubernetes Secret resource, existing secret will have priority over `clientId` and/or `clientSecret`` + existingClientSecretName: "" + existingClientIdKey: "" + existingClientSecretKey: "" + claimName: "policy" + scopes: "openid,profile,email" + redirectUri: "https://console-endpoint-url/oauth_callback" + # Can leave empty + claimPrefix: "" + comment: "" + displayName: "" + +networkPolicy: + enabled: false + # Specifies whether the policies created will be standard Network Policies (flavor: kubernetes) + # or Cilium Network Policies (flavor: cilium) + flavor: kubernetes + allowExternal: true + # only when using flavor: cilium + egressEntities: + - kube-apiserver + +## PodDisruptionBudget settings +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ +## +podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +## Specify the service account to use for the MinIO pods. If 'create' is set to 'false' +## and 'name' is left unspecified, the account 'default' will be used. +serviceAccount: + create: true + ## The name of the service account to use. If 'create' is 'true', a service account with that name + ## will be created. + name: "minio-sa" + +metrics: + serviceMonitor: + enabled: false + # scrape each node/pod individually for additional metrics + includeNode: false + public: true + additionalLabels: {} + annotations: {} + # for node metrics + relabelConfigs: {} + # for cluster metrics + relabelConfigsCluster: {} + # metricRelabelings: + # - regex: (server|pod) + # action: labeldrop + namespace: ~ + # Scrape interval, for example `interval: 30s` + interval: ~ + # Scrape timeout, for example `scrapeTimeout: 10s` + scrapeTimeout: ~ + +## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md +## Define endpoints to enable this section. +etcd: + endpoints: [] + pathPrefix: "" + corednsPathPrefix: "" + clientCert: "" + clientCertKey: "" diff --git a/opencloud/charts/loki/charts/rollout-operator/.helmignore b/opencloud/charts/loki/charts/rollout-operator/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/opencloud/charts/loki/charts/rollout-operator/Chart.yaml b/opencloud/charts/loki/charts/rollout-operator/Chart.yaml new file mode 100644 index 0000000..ebdcb57 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +appVersion: v0.20.0 +description: Grafana rollout-operator +home: https://github.com/grafana/rollout-operator +kubeVersion: ^1.10.0-0 +name: rollout-operator +type: application +version: 0.20.0 diff --git a/opencloud/charts/loki/charts/rollout-operator/README.md b/opencloud/charts/loki/charts/rollout-operator/README.md new file mode 100644 index 0000000..ddf1d17 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/README.md @@ -0,0 +1,72 @@ +# Grafana rollout-operator Helm Chart + +Helm chart for deploying [Grafana rollout-operator](https://github.com/grafana/rollout-operator) to Kubernetes. + +# rollout-operator + +![Version: 0.20.0](https://img.shields.io/badge/Version-0.20.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.20.0](https://img.shields.io/badge/AppVersion-v0.20.0-informational?style=flat-square) + +Grafana rollout-operator + +## Requirements + +Kubernetes: `^1.10.0-0` + +## Installation + +This section describes various use cases for installation, upgrade and migration from different systems and versions. + +### Preparation + +These are the common tasks to perform before any of the use cases. + +```bash +# Add the repository +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +### Installation of Grafana Rollout Operator + +```bash +helm install -n grafana/rollout-operator +``` + +The Grafana rollout-operator should be installed in the same namespace as the statefulsets it is operating upon. +It is not a highly available application and runs as a single pod. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| fullnameOverride | string | `""` | | +| global.commonLabels | object | `{}` | Common labels for all object directly managed by this chart. | +| hostAliases | list | `[]` | hostAliases to add | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"grafana/rollout-operator"` | | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | | +| minReadySeconds | int | `10` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | Pod Annotations | +| podLabels | object | `{}` | Pod (extra) Labels | +| podSecurityContext | object | `{}` | | +| priorityClassName | string | `""` | | +| resources.limits.memory | string | `"200Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"100Mi"` | | +| securityContext | object | `{}` | | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| serviceMonitor.annotations | object | `{}` | ServiceMonitor annotations | +| serviceMonitor.enabled | bool | `false` | Create ServiceMonitor to scrape metrics for Prometheus | +| serviceMonitor.interval | string | `nil` | ServiceMonitor scrape interval | +| serviceMonitor.labels | object | `{}` | Additional ServiceMonitor labels | +| serviceMonitor.namespace | string | `nil` | Alternative namespace for ServiceMonitor resources | +| serviceMonitor.namespaceSelector | object | `{}` | Namespace selector for ServiceMonitor resources | +| serviceMonitor.relabelings | list | `[]` | ServiceMonitor relabel configs to apply to samples before scraping https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig | +| serviceMonitor.scrapeTimeout | string | `nil` | ServiceMonitor scrape timeout in Go duration format (e.g. 15s) | +| tolerations | list | `[]` | | diff --git a/opencloud/charts/loki/charts/rollout-operator/README.md.gotmpl b/opencloud/charts/loki/charts/rollout-operator/README.md.gotmpl new file mode 100644 index 0000000..0ac2d47 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/README.md.gotmpl @@ -0,0 +1,38 @@ +# Grafana rollout-operator Helm Chart + +Helm chart for deploying [Grafana rollout-operator]({{ template "chart.homepage" . }}) to Kubernetes. + +{{ template "chart.header" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Installation + +This section describes various use cases for installation, upgrade and migration from different systems and versions. + +### Preparation + +These are the common tasks to perform before any of the use cases. + +```bash +# Add the repository +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +### Installation of Grafana Rollout Operator + +```bash +helm install -n grafana/rollout-operator +``` + +The Grafana rollout-operator should be installed in the same namespace as the statefulsets it is operating upon. +It is not a highly available application and runs as a single pod. + +{{ template "chart.valuesSection" . }} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/NOTES.txt b/opencloud/charts/loki/charts/rollout-operator/templates/NOTES.txt new file mode 100644 index 0000000..a76e5ba --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/NOTES.txt @@ -0,0 +1,10 @@ +Repo : {{ .Chart.Home }} + +Validation: + +Check the logs of the pod and ensure messages for reconcilliation of the statefulsets are present. +``` +kubectl logs -n {{ .Release.Namespace }} -l {{ include "cli.labels" . }} +``` +Example log line: +level=debug ts=2022-04-20T13:59:52.783051541Z msg="reconciling StatefulSet" statefulset=mimir-store-gateway-zone-a diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/_helpers.tpl b/opencloud/charts/loki/charts/rollout-operator/templates/_helpers.tpl new file mode 100644 index 0000000..68ae702 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "rollout-operator.name" -}} +{{- default (include "rollout-operator.chartName" .) .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "rollout-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default (include "rollout-operator.chartName" .) .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Recalculate the chart name, because it may be sub-chart included as rollout_operator, +and _ is not valid in resource names. +*/}} +{{- define "rollout-operator.chartName" -}} +{{- print .Chart.Name | replace "_" "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "rollout-operator.chart" -}} +{{- printf "%s-%s" (include "rollout-operator.chartName" .) .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "rollout-operator.labels" -}} +helm.sh/chart: {{ include "rollout-operator.chart" . }} +{{ include "rollout-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.global.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "rollout-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "rollout-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "rollout-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "rollout-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + + +{{- define "cli.labels" -}} +{{- $list := list -}} +{{- range $k, $v := ( include "rollout-operator.selectorLabels" . | fromYaml ) -}} +{{- $list = append $list (printf "%s=%s" $k $v) -}} +{{- end -}} +{{ join "," $list }} +{{- end -}} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/deployment.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/deployment.yaml new file mode 100644 index 0000000..d35b866 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "rollout-operator.fullname" . }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} +spec: + replicas: 1 + minReadySeconds: {{ .Values.minReadySeconds }} + selector: + matchLabels: + {{- include "rollout-operator.selectorLabels" . | nindent 6 }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "rollout-operator.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "rollout-operator.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: rollout-operator + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - -kubernetes.namespace={{ .Release.Namespace }} + ports: + - name: http-metrics + containerPort: 8001 + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/role.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/role.yaml new file mode 100644 index 0000000..7bc2570 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/role.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "rollout-operator.fullname" . }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - watch + - delete +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - list + - get + - watch +- apiGroups: + - apps + resources: + - statefulsets/status + verbs: + - update diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/rolebinding.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/rolebinding.yaml new file mode 100644 index 0000000..d1cfe68 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "rollout-operator.fullname" . }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "rollout-operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "rollout-operator.serviceAccountName" . }} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/service.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/service.yaml new file mode 100644 index 0000000..60ce5b1 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "rollout-operator.fullname" . }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 8001 + targetPort: http-metrics + protocol: TCP + name: http-metrics + selector: + {{- include "rollout-operator.selectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/serviceaccount.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/serviceaccount.yaml new file mode 100644 index 0000000..37698a4 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "rollout-operator.serviceAccountName" . }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/charts/rollout-operator/templates/servicemonitor.yaml b/opencloud/charts/loki/charts/rollout-operator/templates/servicemonitor.yaml new file mode 100644 index 0000000..8fa7c1b --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/templates/servicemonitor.yaml @@ -0,0 +1,36 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "rollout-operator.fullname" . }} + {{- with .Values.serviceMonitor.namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "rollout-operator.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.serviceMonitor.namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "rollout-operator.selectorLabels" . | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + scheme: http +{{- end -}} diff --git a/opencloud/charts/loki/charts/rollout-operator/values.yaml b/opencloud/charts/loki/charts/rollout-operator/values.yaml new file mode 100644 index 0000000..1711671 --- /dev/null +++ b/opencloud/charts/loki/charts/rollout-operator/values.yaml @@ -0,0 +1,89 @@ +# Default values for rollout-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + + +global: + # -- Common labels for all object directly managed by this chart. + commonLabels: {} + + +image: + repository: grafana/rollout-operator + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] + +# -- hostAliases to add +hostAliases: [] +# - ip: 1.2.3.4 +# hostnames: +# - domain.tld + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- Annotations to add to the service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- Pod Annotations +podAnnotations: {} + +# -- Pod (extra) Labels +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +resources: + limits: + # cpu: "1" + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + +minReadySeconds: 10 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +priorityClassName: "" + +serviceMonitor: + # -- Create ServiceMonitor to scrape metrics for Prometheus + enabled: false + # -- Alternative namespace for ServiceMonitor resources + namespace: null + # -- Namespace selector for ServiceMonitor resources + namespaceSelector: {} + # -- ServiceMonitor annotations + annotations: {} + # -- Additional ServiceMonitor labels + labels: {} + # -- ServiceMonitor scrape interval + interval: null + # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s) + scrapeTimeout: null + # -- ServiceMonitor relabel configs to apply to samples before scraping + # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + relabelings: [] diff --git a/opencloud/charts/loki/distributed-values.yaml b/opencloud/charts/loki/distributed-values.yaml new file mode 100644 index 0000000..78a1f11 --- /dev/null +++ b/opencloud/charts/loki/distributed-values.yaml @@ -0,0 +1,71 @@ +--- +loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + tracing: + enabled: true + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 4 + +#gateway: +# ingress: +# enabled: true +# hosts: +# - host: FIXME +# paths: +# - path: / +# pathType: Prefix + +deploymentMode: Distributed + +ingester: + replicas: 3 +querier: + replicas: 3 + maxUnavailable: 2 +queryFrontend: + replicas: 2 + maxUnavailable: 1 +queryScheduler: + replicas: 2 +distributor: + replicas: 3 + maxUnavailable: 2 +compactor: + replicas: 1 +indexGateway: + replicas: 2 + maxUnavailable: 1 + +# optional experimental components +bloomPlanner: + replicas: 0 +bloomBuilder: + replicas: 0 +bloomGateway: + replicas: 0 + +# Enable minio for storage +minio: + enabled: true + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +singleBinary: + replicas: 0 diff --git a/opencloud/charts/loki/docs/examples/README.md b/opencloud/charts/loki/docs/examples/README.md new file mode 100644 index 0000000..84cbae3 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/README.md @@ -0,0 +1,4 @@ +## Introduction +The Helm Charts found under the examples directory are getting started examples which you can use to deploy Loki using the Simple Scalable architecture quickly. Currently, the examples include: +- [Deploying Grafana Enterprise Logs (Loki in Enterprise mode)](https://github.com/grafana/loki/tree/main/production/helm/loki/docs/examples/enterprise) +- [Deploying Loki OSS](https://github.com/grafana/loki/tree/main/production/helm/loki/docs/examples/oss) diff --git a/opencloud/charts/loki/docs/examples/enterprise/README.md b/opencloud/charts/loki/docs/examples/enterprise/README.md new file mode 100644 index 0000000..82c0d28 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/enterprise/README.md @@ -0,0 +1,28 @@ +## Introduction +This example gives you an example or getting started overrides value file for deploying Loki (Enterprise Licensed) using the Simple Scalable architecture in GKE and using GCS. + +## Installation of Helm Chart +These instructions assume you already have access to a Kubernetes cluster, GCS Bucket and GCP Service Account which has read/write permissions to that GCS Bucket. + +### Populate Secret Values +Populate the [enterprise-secrets.yaml](./enterprise-secrets.yaml) so that: +- The `gcp_service_account.json` secret has the contents of your GCP Service Account JSON key. +- The `license.jwt` secret has the contents of your Grafana Enterprise Logs license key given to your by Grafana Labs. + +Deploy the secrets file to your k8s cluster with the command: + +`kubectl apply -f enterprise-secrets.yaml` + +### Configure the Helm Chart +Open [overrides-enterprise-gcs.yaml](./overrides-enterprise-gcs.yaml) and replace `{YOUR_GCS_BUCKET}` with the name of your GCS bucket. If there are other things you'd like to configure, view the core [Values.yaml file](https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml) and override anything else you need to within the overrides-enterprise-gcs.yaml file. + +### Install the Helm chart + +`helm upgrade --install --values {PATH_TO_YOUR_OVERRIDES_YAML_FILE} {YOUR_RELEASE_NAME} grafana/loki-simple-scalable --namespace {KUBERNETES_NAMESPACE}` + +### Get the Token for Grafana to connect +`export POD_NAME=$(kubectl get pods --namespace {KUBERNETES_NAMESPACE} -l "job-name=enterprise-logs-tokengen" -o jsonpath="{.items[0].metadata.name}")` + +`kubectl --namespace {KUBERNETES_NAMESPACE} logs $POD_NAME loki | grep Token` + +Take note of this token, you will need it when connecting Grafana Enterprise Logs to Grafana. diff --git a/opencloud/charts/loki/docs/examples/enterprise/enterprise-secrets.yaml b/opencloud/charts/loki/docs/examples/enterprise/enterprise-secrets.yaml new file mode 100644 index 0000000..698e94b --- /dev/null +++ b/opencloud/charts/loki/docs/examples/enterprise/enterprise-secrets.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gel-secrets +type: Opaque +stringData: + gcp_service_account.json: | + { + GCP_SERVICE_ACCOUNT_JSON_HERE + } + + license.jwt: LICENSE_HERE diff --git a/opencloud/charts/loki/docs/examples/enterprise/overrides-enterprise-gcs.yaml b/opencloud/charts/loki/docs/examples/enterprise/overrides-enterprise-gcs.yaml new file mode 100644 index 0000000..01210d3 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/enterprise/overrides-enterprise-gcs.yaml @@ -0,0 +1,83 @@ +enterprise: + enabled: true + useExternalLicense: true + externalLicenseName: gel-secrets + tokengen: + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gel_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: gel-secrets + mountPath: "/etc/gel_secrets" + extraVolumes: + - name: gel-secrets + secret: + secretName: gel-secrets + items: + - key: license.jwt + path: license.jwt + - key: gcp_service_account.json + path: gcp_service_account.json +loki: + auth_enabled: true + + storage: + type: gcs + bucketNames: + chunks: {YOUR_GCS_BUCKET} + ruler: {YOUR_GCS_BUCKET} + admin: {YOUR_GCS_BUCKET} + +minio: + enabled: false + +write: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gel_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: gel-secrets + mountPath: "/etc/gel_secrets" + extraVolumes: + - name: gel-secrets + secret: + secretName: gel-secrets + items: + - key: license.jwt + path: license.jwt + - key: gcp_service_account.json + path: gcp_service_account.json + +read: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gel_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: gel-secrets + mountPath: "/etc/gel_secrets" + extraVolumes: + - name: gel-secrets + secret: + secretName: gel-secrets + items: + - key: license.jwt + path: license.jwt + - key: gcp_service_account.json + path: gcp_service_account.json + +gateway: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gel_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: gel-secrets + mountPath: "/etc/gel_secrets" + extraVolumes: + - name: gel-secrets + secret: + secretName: gel-secrets + items: + - key: license.jwt + path: license.jwt + - key: gcp_service_account.json + path: gcp_service_account.json diff --git a/opencloud/charts/loki/docs/examples/oss/README.md b/opencloud/charts/loki/docs/examples/oss/README.md new file mode 100644 index 0000000..9a0a410 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/oss/README.md @@ -0,0 +1,20 @@ +## Introduction +This example gives you an example or getting started overrides value file for deploying Loki (OSS) using the Simple Scalable architecture in GKE and using GCS + +## Installation of Helm Chart +These instructions assume you have already have access to a Kubernetes cluster, GCS Bucket and GCP Service Account which has read/write permissions to that GCS Bucket. + +### Populate Secret Values +Populate the examples/enterprise/enterprise-secrets.yaml so that: +- The gcp_service_account.json secret has the contents of your GCP Service Account JSON key + +Deploy the secrets file to your k8s cluster. + +`kubectl apply -f loki-secrets.yaml` + +### Configure the Helm Chart +Open examples/enterprise/overides-oss-gcs.yaml and replace `{YOUR_GCS_BUCKET}` with the name of your GCS bucket. If there are other things you'd like to configure, view the core [Values.yaml file](https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml) and override anything else you need to within the overrides-enterprise-gcs.yaml file. + +### Install the Helm chart + +`helm upgrade --install --values {PATH_TO_YOUR_OVERRIDES_YAML_FILE} {YOUR_RELEASE_NAME} grafana/loki-simple-scalable --namespace {KUBERNETES_NAMESPACE}` diff --git a/opencloud/charts/loki/docs/examples/oss/oss-secrets.yaml b/opencloud/charts/loki/docs/examples/oss/oss-secrets.yaml new file mode 100644 index 0000000..4fbf5e7 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/oss/oss-secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: loki-secrets +type: Opaque +stringData: + gcp_service_account.json: | + { + GCP_SERVICE_ACCOUNT_JSON_HERE + } \ No newline at end of file diff --git a/opencloud/charts/loki/docs/examples/oss/overrides-oss-gcs.yaml b/opencloud/charts/loki/docs/examples/oss/overrides-oss-gcs.yaml new file mode 100644 index 0000000..3e94f84 --- /dev/null +++ b/opencloud/charts/loki/docs/examples/oss/overrides-oss-gcs.yaml @@ -0,0 +1,77 @@ +enterprise: + enabled: false + adminApi: + enabled: false + useExternalLicense: false + + config: | + admin_client: + storage: + gcs: + bucket_name: {YOUR_GCS_BUCKET} + auth: + type: trust + auth_enabled: false + cluster_name: loki-logs + +loki: + auth_enabled: false + + commonConfig: + path_prefix: /var/loki + replication_factor: 3 + + storage: + type: gcs + bucketNames: + chunks: {YOUR_GCS_BUCKET} + ruler: {YOUR_GCS_BUCKET} + admin: {YOUR_GCS_BUCKET} + +minio: + enabled: false + +write: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/loki_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: loki-secrets + mountPath: "/etc/loki_secrets" + extraVolumes: + - name: loki-secrets + secret: + secretName: loki-secrets + items: + - key: gcp_service_account.json + path: gcp_service_account.json + +read: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/loki_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: loki-secrets + mountPath: "/etc/loki_secrets" + extraVolumes: + - name: loki-secrets + secret: + secretName: loki-secrets + items: + - key: gcp_service_account.json + path: gcp_service_account.json + +gateway: + extraEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/loki_secrets/gcp_service_account.json" + extraVolumeMounts: + - name: loki-secrets + mountPath: "/etc/loki_secrets" + extraVolumes: + - name: loki-secrets + secret: + secretName: loki-secrets + items: + - key: gcp_service_account.json + path: gcp_service_account.json diff --git a/opencloud/charts/loki/reference.md.gotmpl b/opencloud/charts/loki/reference.md.gotmpl new file mode 100644 index 0000000..0efc492 --- /dev/null +++ b/opencloud/charts/loki/reference.md.gotmpl @@ -0,0 +1,49 @@ +--- +title: Helm Chart Values +menuTitle: Helm chart values +description: Reference for Helm Chart values. +aliases: + - ../../../installation/helm/reference/ +weight: 500 +keywords: [] +--- + + + +# Helm Chart Values + + + + +This is the generated reference for the Loki Helm Chart values. + +> **Note:** This reference is for the Loki Helm chart version 3.0 or greater. +> If you are using the `grafana/loki-stack` Helm chart from the community repo, +> please refer to the `values.yaml` of the respective Github repository +> [grafana/helm-charts](https://github.com/grafana/helm-charts/tree/main/charts/loki-stack). + + +{{ define "chart.valuesTableHtml" }} +{{ `{{< responsive-table >}}` }} + + + + + + + + + {{- range .Values }} + + + + + + + {{- end }} + +
KeyTypeDescriptionDefault
{{ .Key }}{{ .Type }}{{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }}{{ template "chart.valueDefaultColumnRender" . }}
+{{ `{{< /responsive-table >}}` }} +{{ end }} + +{{ template "chart.valuesTableHtml" . }} diff --git a/opencloud/charts/loki/scenarios/README.md b/opencloud/charts/loki/scenarios/README.md new file mode 100644 index 0000000..1ec8692 --- /dev/null +++ b/opencloud/charts/loki/scenarios/README.md @@ -0,0 +1,19 @@ +These scenarios are used by Github Workflow: [Publish Rendered Helm Chart Diff](../../../../.github/workflows/helm-loki-ci.yml). + +Each scenario is used as the values file for the Loki Helm chart to render Kubernetes manifests in `base` and `PR's` branch to compare the content and report the diff on Pull Request as a comment([example](https://github.com/grafana/loki/pull/14127#issuecomment-2348360828)). It gives the ability to the reviewer to understand how the changes in the chart modify resulting manifests. + +![img.png](images/img.png) + +The workflow reports three types of changes for each scenario: + +1. Added files - the manifests that are added in the current PR and that did not exist in `base` branch. + +![added.png](images/added.png) + + +2. Modified files - the manifests that exist in both branches but the changes in PRs branch modify them. +![modified.png](images/modified.png) + +3. Removed files - the manifests that exist in `base` branch but do not exist in PRs branch. + +![removed.png](images/removed.png) \ No newline at end of file diff --git a/opencloud/charts/loki/scenarios/default-single-binary-values.yaml b/opencloud/charts/loki/scenarios/default-single-binary-values.yaml new file mode 100644 index 0000000..78a1f11 --- /dev/null +++ b/opencloud/charts/loki/scenarios/default-single-binary-values.yaml @@ -0,0 +1,71 @@ +--- +loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + tracing: + enabled: true + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 4 + +#gateway: +# ingress: +# enabled: true +# hosts: +# - host: FIXME +# paths: +# - path: / +# pathType: Prefix + +deploymentMode: Distributed + +ingester: + replicas: 3 +querier: + replicas: 3 + maxUnavailable: 2 +queryFrontend: + replicas: 2 + maxUnavailable: 1 +queryScheduler: + replicas: 2 +distributor: + replicas: 3 + maxUnavailable: 2 +compactor: + replicas: 1 +indexGateway: + replicas: 2 + maxUnavailable: 1 + +# optional experimental components +bloomPlanner: + replicas: 0 +bloomBuilder: + replicas: 0 +bloomGateway: + replicas: 0 + +# Enable minio for storage +minio: + enabled: true + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +singleBinary: + replicas: 0 diff --git a/opencloud/charts/loki/scenarios/default-values.yaml b/opencloud/charts/loki/scenarios/default-values.yaml new file mode 100644 index 0000000..a79baee --- /dev/null +++ b/opencloud/charts/loki/scenarios/default-values.yaml @@ -0,0 +1,16 @@ +--- +loki: + commonConfig: + replication_factor: 1 + useTestSchema: true + storage: + bucketNames: + chunks: chunks + ruler: ruler + admin: admin +read: + replicas: 1 +write: + replicas: 1 +backend: + replicas: 1 diff --git a/opencloud/charts/loki/scenarios/images/added.png b/opencloud/charts/loki/scenarios/images/added.png new file mode 100644 index 0000000000000000000000000000000000000000..ced9f9554a8f8c6518a70ddff1e0a5825d9c8b72 GIT binary patch literal 91760 zcmeFYXH=8h*7xn+ZX1ZGNN*|%0s>M3NL3V&CN%*<6A2(mAR>ek2&lA8mo6Y6y(9q% zgbq=V-lT*Os)F=h1BCKK&pG$m_ow&c`@CbkjKLTrWL?)(t7gP=+vpxc>1s3`LpzQ?i9%>o;vmVl-A>iCSKOd^%p*|k z{k+KeKeO%>-}%nKKz)0i{qkRwhovXJ%=^meb5wWL&U*AJdUa;2@L;HZ9DT68u)5`* zu|_%YxBph*UOaAgg+3|z56@5keb#R(&me#NehK~3Rw2n@{`>FO;Cp!IiyCadUwmYX zfO7}LocaBt;Ecd%%l|**|G{F3Dw23a{SIh%HWjg37$59$PJr?E5CfeOE{gMLg}v1h z*%4#oS1p!Pen)SG;XkHMw3fvC>@K$tA(8gd4qa_g%DcnM+5AT@c7^;mh0r3o51kBC z;$?FsSrt82dO)7_DayoGdKK>bB7v9H&ff^U4AtJeym@u=+PZ9eB=%w!SoItemx-O= zsJeD)|NF+S;?X_uu3GB|yt5*Wl3tJaYh|R2nLg~-*u{>+fVx1Xpt>_Wx2j`+Fh|8y zv$+0(^2AW76sZ1g{f2{H-s4~QP&=RLa9jTd;!U#h6C(l4M*d>WlAc|ywv!mF3!8Eb zguM+_|JUadZg?|#U##I2lEtU!YE2u|+roisUagAwX!Noj<%%XxOqzSY7UlbwfHLV` zC27TtuPkK<^pb^b9j0;b2%OMgB$M<8sxx@zv!lwt7r0pbQ}QR^5;B(496g~i{j^rG zbf9)(r+7Ja^3cI!hh*x=%gd{Bzy&I{OtwOjfD|(Ec*i9boC|akpB_-+itqL^``4um zl4D!nk>v-q93VS5P_i0=jeB8y&CkR>EuWt;n!YCE4nG=$Zanq>fD-9!Q=#>D0+KKN+q@_ddKht1 z{7nghjb-_i2)O zCK(07|Lqx$wh-EIo8O{wMv-LXXeFXo)@96$Pz;2i$R##5cJD~tdjlEA?s^LM@C7C1 zcI0#{Y&sF%+__XX2dd-Q)YH$Azm_y*GIs3sYjYR*MQS7NT}IwX@^L>7Q9wIDCQpv3 zj;V5p-CFxrUQf&6V@j`|Y^nkhI?pzCmjQ!T*-GvWlz*zcW5R!HX?eM8g4#eO<(v}; zM_Isz5Dfb{TM3br7P573#mZR1O4kK4w!CD@z%tC(b%{fT=hsq4EbNo6oXxtl^NLxc zf;n>e`{bqF+P_q`Gns)wv{d4LD*VgaGnBR%fii(;B`MnE3C-V;fA90Tx5vk8Cn~hv zN+NFqKoK!fmx35-m{VSB2jtGB`?%<#bnUq*%Cx<|n2UynR%-F~X(fRn#eARre9Vo; z6oR92?G9Fctr;q@1LCy3zGxsyeFmYmwfRl4WSk%OJ^HE#o@?4Qv8aF-rf_KcdoB;h zpw|QVu@^)WY$0;+5iTiHMC$OgzS3>lh};Qci134y;+$tHhiDJBMyY!=ug3sJ!K8l2 zbVgUb`stn&=anpnVl9L=GYA>5fI{kYSD5sdx_3SLkvTz5c}Yv7OgUN?n6DAs$mzjn z;(cZ(GP?ce`%-r3b3B`Oko9X@3gUh4;n5I?{|Rl9_Wh6KF1oSn0Y7Rno()@RfN4l% zrv$N1ux`*;04C_sG>bbYJu)#NSvJaiE$pN>Uo{E6Je|>%sB{E3BUdlZz&m&c%FZw= z8(pYGb=O?hutoFW7K@Y1)GUcgD-kKq^|(5>?pi^#80qS6GoyCvVCm7pBP)x_mA2Ki z6N8~rsB?G4>by^D`iIC2nR*8_wl&={<%pP(!Pe+Psi}9UT^O9SerL5F@&m9$cWNfx z9Uy0>;z-@hP6~1*=TYWZK+MW%0a~$Vy{JV!iok6BL&xU8s9R;6q`IOf(% z3QR&hhuE=T{Ah}vWQUq~lr1HkA`mT6bT_r{aQ}jk=@65NK&oePEKv|G964%`#7?wM zL6Mi2g@g_jMoVFy-4*y8JAQ3D&6K{HoszSUyvbM^v;wKQ6M+RkE8$nZ5xD=umH4&o zX5i(b0&C7M<@Tdd%P~T1A)f7k>=yeiHl?c~E9k1hnv+b;CPiJ=GH}B>KE`>!fopz- zF9OcbK`HT_5mk8+w7)e+4qPi_6StgaW_kC8#9_hTP^(dqce^6%6&O+pA5ZM||DL(r z(r1UKvERKYrZuqo;mXAiLCN=FPd!@GJRMUj;oa$$b+;6kmbU_`X+!>gidcW!eI1|% z=tp@kfIR-byVn0&LeIp-#hFR71`XmuVz`>?xmqVNNLy}gcCF(e)N{61}eCA`+DP!mOdaV8I(ZHH= zeRA+=i4790%xBH#9?xk1*3VeSGIKX}o~nGrWcHqZ6k057M`czAxIO2#<#dp3~P4DQ>1CEuE#;V*fYERA0j#I6iD zbhQhPyLt|2fCv*JRWD9VhsbQN+>hMrXUB;5ERf%+ya?k%g_6kBBG_Jnb*|M(D;2)r z)>+0$?UBG5^zHb0P`=O(=Cg7E8zYkJD!V36N-CDDMg91D!$nO)d~mcy|F!cs-0u_` zaz5;@)m=MNuejZCvLH-kI>~U|Ri|J|NGjpA*$y37V<#%vZCsHvR>z}mC-2hyhJpY; zQb>=Y#Ti;jv@yg|Xo@g(``0tzbDOn$pSsG62Q8c7Hiysk_R_V*>S)w8Y6#;+ew8{w zU(X1vT#hHImE5PgpYch2wMcQJVw5>p5%I)KR)Ds$c6gn3$WF9mB?C=uOx^grV;>&%BK#`_ zarx=TsoFn4bJNpFE*T19a0vxd=kSuyMC9?wq-g_~WnwB8(nSz&Xo;N~SznyJJUc^J zY8e2^wWYvo*|jKM;u*+TksHvFF+>XB-M*i>0|%|{70 zJo!oT8E$>7L9MZ0wiZa8gCeKzilHbiw!t=Xf^az&ZXL4K=r$Rwr)MXqTgYP}(LBji zT@Iz)ZYp8)506-O>$EPa%Mluv6bN*NKypWxb}NMAA5s@gYL>a?mCAdd-FjheUy*iA zCmv}%$44^>@)dQ~sl~WaPlteBV$PzOLPb|eP@n7J?v13V^CuR56Wb$0%Yg%armRov zuWafnsGa9!fSPE$6X*0pds!DfC#UB_V@NfnER%k!TBx2P%kPtt+|2RSCK}(4i@LTG zI=t;ez4+^mw|juyA;f}kV#ss3XT&nJrsdDAL;Z7cw{ z6s%R{@n@Qwy51Sg1SbPVM#=5^c!@IwL>M*5UId=#l8s5JG8qb1*E1P&KGf*q#ATi(2Sy z?wnGZt(txmo(Fbt_9rZhL~ypmJ-sjHJMnfQmZajJABu||2l7xQ^niYgj;h+v7yp< zwq!`X7s*JN1@jW>WWrHvQ>jlB8jM}~(Di|plVaA*Wgb-RHdOagbvtWlA6j3rl#5f^ z+%oRHC`J(pEU;Iw4CrG~dk>1hP65AYkSFuijTkXeB}5S{)^g{S*--tEhn1b!QcdBv z?8fGC>IC~#;&Oxenb#sxCO1B^S&#Pev)l;0bOslABi-+coyL~h>|5m*eP+|F?kCrV2vU5pkKf+CvR+Kv&vC3*X zd2IrdcqxJZnnTAM&v{E@5p6FfRQzc||~DFi=dVa_p53sdTxdf2yhCIb^Hl>j6<^ z0IBKF0;z+KE7L37u0&?XJTb)B3pm7PL1C{c@xmvx{WU@{s{jT~>K!#P7ZUrB}k56j~s)^N&*lb{6Wx1x~7_kpsxm_5fWRIe$aD$+NO*qVWkg2uiHioe@ zvAa#gk1?Cg0Y~db^j_+0Dyb2?d-8qW?(HADfFSD>pVy!WYYArvZG|>l!DhxdaENEI(1}cQUJom-~+`-YH}?w1VvrtrHmp>>Njt@cPHa zbq?-kvc@?eJ4R{f*v|l(dY&H6UFzAvWCXTmbi~6)tB6*FlNKo1ilg+OR6ec7h}5Ir zSkxsZ?#<+Zt#HP2c4Bj!=0RkIlg&OIub)hEAty*1UpW!!1A8uW){9d}(1X5{@I)o< zV^i{ye1ym!5R)B$Q4NoUG8_?(oJ5}N5Nm>id`?>5Mz(`TfS1dBJvwooCP9)-I+(j^ z^9Cl_Qg21PcOpJ0tvs_w#lVL<;%gm3<_gmZ;Gw&FBQAL+{;c^TegrJl+VHopXyR~1 zAA6+nV9-nj%9WznbJ0Ut0Mqe|$*B*I;&DoCtg~9!cQ4=pY~i#`oGbav6%vCVv)1c} zcPMwiXuEyR=bBc7?qx)rtFwnia%0=>n;5Io=RwR|1ypBpy7uJ9ZaWIz9$=daz!Stu z)fdgfv0z%4Zy|Rv&Qvr9SBb=NQTF7a?SnNz%&KgO7Ls^qWaM;>({{IQea#_szpJbl z!!scB{3pRSmG7BUXbAn#F%)Qc$c4e2kdEY4f=wr?wKo8L)${&@juu;&G@0AjN-a!R z+rdUFI3HvyRx)3(vWh*D$s5YoGqNrulg1+8N-B@V-h=aDx0>=!NLFq1kbN$wW+&gI z^P{J7;DZod1=v`!HzN?c{+@I&3U{`~nAc6{8p5BrR6B#s*wTwwCjr5H4WB1vB2bOwGJLg zMPf_P#FglvYq<|`Uk)a%4+L4=;b>8fikQ$rN#wZ6v%X{3BnM}5e#mBz7P+Y(&U!&5 z!~ED{Z$1vts(wE6O|gdkUqG2f&cgi*hZSZ}tDQ9`&L_Z^i~iENvc8WUfSpps;2u#O zlikqt6>kmMRmt0FQuBw)<ZCD*PjS?+hZC&x2no*i3<9I)G(1Q}hv9x3&Fo})xXVp=Wvg0#Z)Uf_UKqv3kjf`I z9+bFv*Y|HinQsusSPE-Hk|md*qRGN>a4S&_knQyJHBQR?txH2wS}AQ>OLZ|oMkr4b z0d%SU=jJEuYG=w<;Y5PCMWq8LWeLH>B4~r(GCB-TQertKR}JlPSbNWjG?R1gi0et^ z?#x-g{1k-a6cAld|F|>9TwT>MxX4A@2B5T(VDxC=QtYKEeDq|X@ANCZ|eJglkUL0?e};V=(Y z?2{>v)36+M#wJ3yr;o6#=1xO@-8fKfAjcMChS-rzkLY`crSF}zpWL`qp@YR;pwt0_ z!499%ZS1j}F&Bc2k1XvL)%m#+=LlKh5bTD)6IO6s<~vFQ_?Dg#6;ugqJW_6wD{<$z z)TQm$W^tbX9dzi0+$7&`p93$jKpPWtod)vuo6ey6Z;DRHmTjDJBmZg9$YGx9K3}kQ zM1*hjJ2qjzY&?F%*27JyT}$RooMvlth{PsW>xirFJuta3*#9FEdn>L02?bPy7RodX zW`rlDg%A!qct#+)VK$u}7hx+VdTD}1M)K0s-&=D_LY#Hi884i=UdSncNqnB{f+m7G zQJ=vz(;&EUw#y?2$aJx&Kruh}kgyBvc0_luZd5(T@{B;h&*>=OU~@~$CKuu+%kn+P7C_2yW|Fe46g;QGA3Rkpk}=k3U3G3{_Ue99fQPFGJc7+^3Q z5M^N%(w*Zp_DQ!s@+1c0POw36n!^&(0n|4g9#-t6*A{Qo;vjb-R^J4ba_!NJB`6t9 zSF$=8L_q5Xp7%PJJov@B6MNlHroBw$jo*8mh~H22V(%80FXkTtF02>duP3UQ9?UcsH z3c~S;T1SKnNsH}*_}I(@r;7IXx1?jC6oL$<7SE~A>5xj1U$Z1k*u-@fb6AJ!Ok1sZ zg?09iM#R+n4*%WZ$ED-b_pMW;HClDY4k{B9z-eCpdi#OgzzgPuBn9Rui#`Y=m&Ehn zmYVUlccYIVgddv~zRO3&@fE;grUt)WMroO+z%2_d=Ie>y+RAoc^in8p$-kqUF05?} zAz(}8z#w+G9?vZ|F8$z`BsK(O2XA8k`eI5$c>V?vwSfsTcNPB7k_DDS=xJvj$y6sK z0=*1iI9`h-RA-w=n(TfHukP-+?lf7zT2lfaK9+Xh6*HXF$~ZnFtp92dSlO6 z=h2rZoeq#(FEy!DXG)ag8PQaE9dA>+k7ARBEffD!P(4Y38}w=euNR$*Ks67`8>RlG z#hMBp^08PGPt}_YcKexZB$gWUpJ*gtrNV>%esTPlm_=u?-h9qxP{%#l(+!;F-Iz@K zdQ)uUFluYQK5M#A3g~2f23H{mgDfTZ9Mt}Hsh=1bLk~%b3I09xm^BYLHT~{=L3vZRM$Im z4#dc@$NTKd~qwN)9cgPjVA~vS zgPH@jQsg`P3&Sp>^JOPAC*Mb|73z8`X~?cA%xH~&0=~w3nY=bS{yP$z1G|$g9#C}R zyo}8?_b2@cbLu}XLLy{rx#)733B9;}4>4KMIkmf58u~#OH9O*C*lS*ofv4)NUJD_= zI^5^8`B#9CD0c3W6kF11i&q?3OJbAlr11=narcw&1Ecfzw!B58&7wuM#QNr}m9QdK z-cXa(h2G!kS$VB)%l8|DU27EX!lYFp(zIoJvs0q?fB1vm`79GKV}>4x|9yM33(a_;Tam~}wRb;^^5*wK5n z3CsZ&R4d{{PGV;rB62rDCj7bnYHkle!e#+nhym6gme`SNLG?FsrHi-Q;%ctVrP*N^ zCXd1ec`;bkPfu|_cSCQs*Cg$Ww+rSLP7vs|$)kt3_$#-7r)UTQ%4y;(vDlm>sv{HI zA)!8(>AaD|T<5NPq5{0_STAL$OfXtqi2iM%qek|I?*X81DwF#?p3q$B6kb!rdv<%(GM0dkriQ+HNeK>4fXn=hjRD1lId-Bk zq)kwvLpp1I)Y?bvv#^qSSs%K}U)IX!bGHk6m@9!6_UZMRFlMoiO$K(trGFB-5;M>7 z``vP{B`r5_m7pfxg^RG<%{W9|mtvggiQb z!vxbHLZ1>Z!*Au1lTy~BAgbE{IiM2~cEL9z^*)~V<7K@n+!yEQI{R^+F7D#uIhFM{ z%Zw+7y_BkrX;=j&0W?`Oj<*ljhn{ifR&al@+vC|#=HQQ%c0qP6r=y{fE8gLEx#UM` zj=nE@*Y9W1HEvzrQ3U)YyH*e_#L{j(e&gOe<5Rl`V&; z9@>%)j%ZXzAL?qi$Ld(J6f!rc zAWxW%tH|mbCE35|#DS~eUzDXF>6e`U0zxEJj#eCW zmztu2Ms-V)()RLFU5dVLh3fm>q|Nys&wW#%?M+8JZq0Ug&i15nx3p`A@Oqf-{$Tc( zFaImNJx<6JJqmJ_qeYrW+=Xh`g2o~(8s&NhnKbtEgp7Vo=hC40!7u9f=;6Qk%gn3j zo)d%o-U#e=r9Z7QQ}NS{=>pKVUxA_5b^#1b-qY-#MKn{1EN0f`n> zkYFT0d)4Z2slakJ+Zge%lZb^k{nFEa;hh682Sr5|-*4Kmnea@O3GG<%V)r|W>;5Z9 zx8ylyKFd&3t6%_U{A90=Ac~I85{|Zc!K1-3|7MLl1Uj>R2Fd-!4}sIB_6n_0ys2q! zJ@+YpUlC};w)pXvV2W#s7T-h7$3NLVy8X0ynkpO%e#?{$hqX8Va}Kg}S(CkCJqSKt zgH%cR>|8kxVwvb{A&p1^7+*y=Jl>^U(;82)`^B$^gkzkArd~3|1yPSN<1LJhp^_9bY&kk zOBSK0*bCG4R^c|_(wlde$`nf%cx2&b-74xe>y3<}v<0Q(ZT%A&Z0kQ~9;g9w$Ju1? z`Ye|MG)m2nc5F_NyR;j`UX{(5=t20_!CtFsIuobVEgI%DZMcT6*wv*a78idTA6}c= zS@k3;S`ZdjB1-oPn-iS>xvS>0x}a293R(JD0#wYpXPlXoHFzlT5h!#X=D5}UbN%>_%* zhFyB!KY2JS2XHXS=(~TkJy0f;?v>>*x=bni4a+?p?2zH0@K>Vms(+8Ze#R?OMG9Xi z>}FX^$ELe_qYgLRez$f!U|8omJ=hkO9?=OjTggVVYGMRI{}}ZL91^)2)k#vpMTGAB zmAaM_8|NQmI*Vn!S?%~`2aSIL#0pE2{?8uDt~Jog)b&+n0%W@BPkXlA3~5J~C(~uU$~InfmwS!BGYZ?*yoV3Hqv*y< zEn}hgRUF2#MsfjK&J})aoM85I*9CeN3I!3AitAT{i}ob=P=(BYhMbq;Q6Xxj$~x~p zTzb`u^?^;v|Hv`+{Z#S2xYiMryEUX0hko~Y#dx*gzR1*nwUCua>+R3*mBoK z6Rk=czdvUmty+3#*)$qoQrfj$J;z*lL5R>X)vWU>qbv`>O0Vwi{Q?%`*?(T^6gC&%YRP*V^7AQBBB zkd}!0=5simbjcT{0zR7(`WWqezwC6axctEn(e0_&hM#?{d?g(LB@H$0=L&WDju1J$ z{u`g3y$HWU<2ugCTQ49KtH#lvn&rr%pYH{)hDdFo?Yw z`k}ViW#P|li2*fFTgV%#QA!=>oP5F}A{aIz!Q&B8Q9UU~xgG$#r!KKfB&{+c^?LxM zm4~{OvegH;f6OlAhyf+gBIYmH93A`sI{B_=2B|!;3S0FrtND@Fr_%8tnQVqDRr!#^ z_B_b>Y_b{BZP!&MJA_STIr}u-M6&-dkwe6CQ?wG;*h}G{wKY*ic8ETyrU$b4STHT+ z2VS%oOEHC-3W(atAV;>BP4hFZfX^J6Sw~W~=cJH}F4rPDH++0p#leQ(Gm4 zR*`T4x`~b4#8b8lc*;4%5M*x{-TlEFul@OS-&64>E7Z?`8)bgiSaxO(^CnN%vTL=~F0d(S+k zMr+YaK*KstFBzp-;jjYcms#%*WBoDQRLo>nMMbIP5gA)*)WK z9QKkf@|ctw59WFw1v3Gu)->)Beb)t41fQGRJ$Lmo2~jvtZ%0(rC+h@Fa8GKL$fXR& zD#*{lKfOsLGH4SE7P0Y1r!uGLh*C`w&Ly&O;^ot7qAGC79iQrGHV{ZqBlA&{J%Bb} z$!FXGsF2_GiD=^88P68|v8a67?$8ZH9KY!vrBau!-ZDZcG06pt64ezzn z{4S&-lB)@D{ytj>DbVKBDYQDpI$QN^jNW5PGN!Gq==yruw0@UZ7i*@PCJ#u=R()^7 zQDmOOTcaX7Tl(5%H0l;>qP@XVrdq9pN$WwSK-9(Hq3p&~X4+;70NCk~?rn?YMZgN2hlVt`otlzuiB5US$(Svzvc&<6#BH05xMXT=%tGgqRwk0#?SFkaU;pMl^=~bzO@f4a!P^@W$7Tog&XqMzn{Ac# zUZXVAD@D)3nidcKIJM3Ck2G{s0Mm%`x5ez>`3mX$=-rTXrXX^oGdqSeVun$|`waUe zx->mTd9IETI^CWlDCX^oEcS{Wvvc5SEa5fDMT(V^y`7cNL|c3y_~Oxh zcCB&SBRTC_AAQ$Qaqf zH4WCpofajXj?u`*`O182HNN_3-{G5u1m&l&3F(G;^v4NC6}UCx!VgjF_{vkK4#M#D z8vcI?r2dVUHcgL7^P3w7NPOQ4vHaw?*`T|TfYEyQqsW`nHv*O|Sas=-dsB+dPvnn;3q~-M zHBWxV#%R$|_p9KPWJFaJP^bYZ7g_7V32wyZbY$PkVn zB#_rqg$ND{H8WP8T6v_BxIut)UM0|k9_ zH`?Zgo+T6=pl@3t(9@sGe7GaCfgUICZLC^S@Ep$!$)AZ)%7}4{CriXx_FI{I^#*#b3RxB7s#h zCI(w#)l$B$jPF}$2loyx1{<2ZG`yiNrXAMVAZ&gO6a?Y6D5U6a_49fHcf>+HlrY_< zjBmiF(HxU2WY5!gQ~foaa$%#?6-(zvU zWb;WqfQ#YQnUOll&Xn`&)Ye;Wljm)QD@yGefPMN0KP^~D1D_`|hTDohV`U0!vu+tL z)&3!J;%z4={o*HU8}K9b#h*O<;(WcP!R?!QmR7fv75LFLf|ndwNPkPM780%N?IA(E zB?s}5hAjZuj(d1DqPqX&qLy9UR*PA_6uyI5Dqphb{f6rcDdlLntPQ!gEkC)9$4+X> z&b}PUk5Aw`8bI&s8m)xMs(868iJR^`nukv}dM_8wqpu(4*wzHAU{9D@SUl-@>&!p$ z7M&GALyx-R)8l`^i>SIpA?fQWNI7c>WAi|MguS3x$#)}}X@vD~vksJTT;u`jbn%8s zFu$qyoRxG|{iMchH@`&smugOdC8CXaiD*tue2cB!z%-*>m=HpC0JIV0ERCNiIMB+d zyYY>%No}oclMTqY0Ji!ZS@);+M8rsEq8R1^J9~i-6mPQdy|Oj#kLH4KFCOvYi^9|C zySXtt8cMQT%JnUh^se7b7=}#&ak$vt8}Xr|ZB|)z>!QP$^YD5T@07md{-j=?bepFm zbv!sInDeXcVtqiEL69Dg=9t=I#toxB+JkwkpG~a^jVZhogha|X=c$jC|GTQaZ8N-; zBZY4ODQf_zDrun0ZA(%ZnVI&GxC__3&m?92>FhH6wq3bS>u;g{4xMOygkjAz+Z^5h z83I%;`A1~g70zT?D+v{2OFRb2x_HV_kD zJX?@{N~&3;3uSl9iKm#J$kl^2#4=UV>8Eeg3hE$a2B}A-;^)&_%5oE3^=Tb#i_!{y zs*wNy3KePBA%mCXQY@T%n{L;Dq=>Heq;#|$bll7wQs>vSRu5TAwCQat%DvBwi*E8h zEphni^vs6RApUMskrMX8#5UjHmgJg(oD^PtztP~#10EHZ(%qI?#mVbuPu)rVUum7B zKzKI8I<{5A&P9K!F>tB>j0`GMOG6~U)Q8kkOkWCTC48EIq{aZ*P5CuN!z^~`3AI7g zS~P4eyek<|+8wtU$Plfb;j^*V=m9U^4vqCnMTYLiDN!H>$H!g*m_Mu5K*LXVz>p=h zOH*o_6kXpLCV?RFzZX2KH*!tiU}ylw6W7s(iud2K?C=p^N_L{}k++q#F$X1UQqNqY z!upI}Iu|EJW4`)YNVH=FwO7PS=+x_mJa|bT|CctwtmUo*I)6=l*HB?*^yPYbAwV!8 zMyx24ol9tGV2@s?)w=B4nWb$0tn6*@boo0w;jNCSZFmIS`iIohq@+o{-m0;PnC?_> z5`Xr*ncvh^Rw!v_&_C*?z6f@f)l2yp^_7lSSei#6*d@Z?0gbNO=&?=Ly31vI$lw=y zRx4>f=I;8WW|h2rYIzC0JJ#mNZ`bnjFOQ}NJk!qRdIl`CHqZ_-c_~|oH(L^~QXffg zHO4G0`>!n?OB}8>Mem$DHP85u^TkBl!Jla*l8Kvwl*jgX3rm^%CY~=^+79%^gjM6;h$xzTCPSX6pY8K+6k@g+ zj*7&t+g+%ajIQ0-|Fr`2^QL_L$y13N{nan-rmjmL4K0B!O|G*yJhRA)AqDFA)#eg~ z+~?8{-l7m3=A6R*mKlg&Zi#|INUYXkxV)@2+tx9a+zMddx<*C8&3kJ+=kPJWGdO*W zh$8Z^xOnHs@uKhHUm#@wzLBW8qqspF$fD6R`RG40c?@Z}SW@H^ZNy)=gwbSTxULT2 zJF9ousF%LxGB*a*{V0g^Ayg{MtG-2%EtBh2q?WX~E%x{REpeJ7dSnz`uM~ZW9^ma= z{zY)Wo|zw{0Wa@pUTw8yHY4@e#J^ zH9f_Owndfwl><1Cucl$6zKksszgj34Ul@sP^f@h9$cWa}FLBk?kHluY)ayvKQr5^w z3vFuMI{y=yMv7?To16i1wP|?te5eMDZd+{ZQ8hRla zM?ZtG7<$Y?>1PfJp7(>XmjzBQH2p-X?&LkBzXvoZ6YtfqhQZ+=fb}54|Jj!S!;9w( zO_e7Zpmur|!d2y0E7D_j&hW5SU8~fbZ4C+6E0CztYOohC8BKNY>3L-3IsH;LN*=*B zQTMh=f&c*D&JPWp!9rnUz8ZWaZ)t06V~!t&~OPE*>7JT>aX_&lDy@bJxW?7WAsdFooHo#amid)oc)P@S>Kqu*r@ zkvPIQ!7Xrt*4JX0zw5Q0`3*J86i^rwlVEcBn=Ja(+##blR=7(`wt{4k*&mWj*DMUa z8)mADwKO!}cd3Twe(%>FMrn8q+Eqe-GCIXFWEWlh=31^?vW^pf7rCy{!fhUtLqA9JgYD)ym0qly18j4=uBzk? zAdWWu*O^x?Nl19Vlx`>aiZ2bUtwv3Rqgcgu*AZYx{nQkih&Qx9ql{^3ePZAIS} z7;|ISw7u-ol90*(I{4K(^TbAeb$%Xuf&3(8WAMeljRY!4O^(K-o4hl%y(`#zIdv*o zdaLD{Y;3QPpQW|5$bsz32stVGrc6HBaTWc*-?~C~7YLe6bRZSe?=CqNnWLRo^)@@g z2aFh+RuY)npb~*j7rprGt?UegV@O`@D2KXq$lF7=UR81`iN1#MZXJ70N*!9Sp_%8@ zM+xEo3Um8^M$6riTwI)G{2q(meO&PAqgog zg;b$&%%1eao%NtzB>_?28>YBbK7ap1`C-HC)}b5{Kke`aI}k?)ip^?ETt{qri&{U3 z-j*%OJ7gtL`jw46+9Y-mqc)x$erJg*Tq*)?!C-aB@I1{j6TBH{=6t*KBGYt`!cEc~)Vv^qcNyOCy?ne(> zxd*(d-6hKksm(9v`7lD1CeROPiAVCzZmCI0AiGPSEljylqk;lGhl)F%|V zx$asHg`^&hzu0IX7t53B-Va4lJt|M0|zCNmv*>4xtVWyjWV3wHPNW^65nu>!er-Zyz>0g%unm zm#j4zRCYzgQstzP(bh71>KjMvMLj#e-k+6kGnLOff$st0~A=i|yC!R+y5`}lmC0@kcOf|F(hNohb1H!=S&7q|kNl3jEq9%lR*86aIkEKbn(s^`Nj_FQHHCug9I{z;+LYb)d zIm%mpKkdi2+FFCO@-r6iKAerGvgXyEam~)(X#22O#~PB3Ntcw1?o`=X=mn;i(Yt61 zZkOeT-CL1^4ss9+8%bnK_Qo-9@h}5>h^WtfU4(h2_+Di6 z7K+~;i3Bp#XKO_7q`EoH%8{+6rx%=*OfM0OUinS5fitaVcY^FY8~0(}hk+KDMp9D* zHs0|y?oVxz(YAv*EU&06PDR&qp$F}?TRqpLa=@fS@Nyld`l#+1nz5&1Q19uT%w-~H z=EmhzYk%TJ8{L3V+4SQlz9p#-dx2JQ+5W!p3a^dW<#cl@NY_6?EVwvB{D5mtAq6*SC}g6$0>Ov8tEcwQQ^YmIu8yTt}6KE@rxYtXwu{Yij!;n z78G#iwH44*hB%Mtcu+HDsi0h>+uN!=ux5MVc)>bUC!Cw?)NN#H-_4Or-S>wy0 zX|20EMB(w)T%C;KyJQ{X5B!i@*D_9y<_``MZ|9n}8q|4HO~%z(lU55ankhdLj+YDT z3aAN(rlyNQ?{|h{k|#eXhdWFv%30LzCa6tkmoM?D9t=<4u+}fzkO?S`+)eYP4Su^i zAGu8Ab%l6MwCq13KcvAY8Wb$8hB5#bR~YAn?je~A(d_fBaL*Q8bhic7AwmP~g%UZ@_p=m1G@e7?<1 zMjFPh1G}`U$Z@Jh)0_@<0_adDIeF=TTNB72?wz7!Uo@E; zb|Sx7U{D1$2`Faw9lQNB)-}u~_K|p96||PYZ73`MZS;`&mu~k#&*x3TYn5DkQn%Fjys4TXGJ;&Z$z<5`5u^j#~(rb>~)`aU2jtg9j^L1UVF>tLQDv2 z&lHU-THPemfNW^?hxfYqoC&Wc?sIUz%ea@`oXtzyJ=mEXB>@YGNf|ZrgZ(cV{$pHT z3o}iP=zH-Y@lSbM8)3DZRrD{;$^n~&^y=-vKpqwT4zGoiVK1=R5opnt2DUZ56D@4j z!I)*3GIed(4Fno|;$K52w)sPOaYUYIBG9B z2y)qKfN;-HAXK^azLD*70n7y882mL)-KoN*G0&~1_M?d9;fLNvxi-ZSZIoaRefmpvDXJ+0{_bsfxTX-C3;XQ>Fefi|#zy=hoyMQ#xZ z3buY_PESc{z!$JI5rfAQ^`Tp6X(~8@uR~LX*m9?YHe_1g#2Vd&v3&%C6#em$dS?K( zZThd~2J}kmiS^c0Y{N+tKkXyHjH$>vanS+0itw{c?Mkukmtusv4(beACql)pphBVh zQ45~IBlGk!IY=J3cX{`@sglUuJ5j>MYv-4+-##|hF%%zQl5o%nwCQc`lblim6h-8N zgwnzcW@c1yk}oQ|F^It{OlRTpz*?PRU|*n&cfsu^KkR2svWsD_Xz>9H30MPX98Tn= zuo&}E-^U_YO6H%q9z_kZ`WZvQ`@-+Y6{K9?5w@ZpC?E=60AmwXbK7-8-RX!n7&#P{ zL~k#yG7Qfzu4a|RAzOPMdOslRhy^R?<3y$Kk(+t~?o8z;LF^|t#-wxg$gAH8mVAvr zInA9-b;5cqT8ulP=2treYA)6Ab*P37h?rrYeMn+P_S76Jq-QpkHv&=u+jj{v=V1v( z=}w~5CNd~;?Gze0he+!wJ+d|J8O#?>DXVto5vSEzC7oyxpOrgw;Cm}#0e!2c}Q|i%+iU( zFYLtsjRj-!M)uV7#*1P-KIHx|M(lsldzo_}D4h#r|KGx18LI1=`mow=a*LBgRxU%i zaTT)K+!pi-f@3mm-c>UdVelBNJLcO;nNQ+H@ytwlRY%!14rTJz7APs3D6g?wlMj8!X5_AS`Tfpc_-b04j-=#t zT1SwM$abE+`)A+9%Xt)((7D$n#nH~#v_cN=&whlIjymKaie*Zd-?$aUg$eBRm!N5z zQz@(UK0dU?&H=?OT8!m0{uLW0=9?uDihS;r#rXVY`OED5Z)Fy&F7M!T{*lBr77ak= z3~E8~MQ)g#WaPmrn;xfQ)|<7QbkR>jT3NEa+c&gSPX16j4Eh`Y;zsk?^> zBR1W^c@NFEAdKa^GCi``>l3;xW1Z{Oc}E7irK_Vq&=c>mHTxO2uq$ui>M0hHA!ihhd2F^h7yPTl1VWg z-7AkpUwo|f+8K5Vg!MNc@L}w|R?67S|H!IY2hAr(l{i2Hizrj{+=c3LCM#szT$b-y zzUHjdE2Oy;v1deC6c!QonhJ*LBhusSP)oITHooJ2QDJGra4;vGXIDb|W-3=I;W}Gh zkuv#GtmYnjI!MAMA@#I{Q&~PdnM?oSpKYJB;-U(|9Igbz^49gBK8XPR*Ba6J z8wa4o_=k#V4hAQ4jlSxNR^r~tJV2e13WCd;SB2&-yTRBAbPlwT#DY$r>IWN>&2AUl z6HQTB@g>S?h`A~~K*p3RA+I%lC=k=;jKAaJ}x>n`)fo zF5HecX*FX^n{TE@fi3^oCjP`Nmd%s;G0tJ(HsqTCH1lTWkg;*tjOIhIFLdZgwGWZVw?m9KvoVd(vsBJV&eiKpT00TC?a@iC5$9*4E z?)`^SJ&fY!H+D%WYn68j!InyN97oB`oag=1CQ_`Ep2c z6v$Ux*&Xu|YBf}n6)_e6Fw|(Up~+4e1e%}4^ zs{V~2(sg>tC*MIRLx!6a-MuW4)Qnhy}_& zo|FEJTgpME|1GfmonN|NT)stKdfShXK?qfhW>@UH%PwNd|D5mXG8*io)J~{?qt}a} zK*VN$Piye+5@4?hLs0WLycqIP()G=E=pghF;*$3vr&ho^J^%{-5P9a+olk&}5bVLA zc2#xmL;tr(Nfgm8t6Pq#33+B0L*7vu(vbv6d#1V%Tn*Accs4QPz?qNDPe-V-Gtu9G9uafx-!c?s`e(?7c)NJfv zGLAqd%rnYYvIK|pj)h{vykY?35`NvBaZLZRQSGzZFqkAGvjACUL#q#6ZPRS&{Q~FL zEZ8c@gzq@hL8Iq3ocn9F*#7z@;Q5aNk>lyw=s z=Dij8u+trz)%9mQ23!L;QLr^ca0ME){3YMX->c`)=G9njuv_?r$F8qkVW)opJH8rS zxsx)epfO=|qQEaZ_J&NGw6-wNq9*Ri#LAoRT$h?v-bu({#rcf8-kqrAyO!7yl1r)D z!plQjBpytDKt3 zK*wInWG$Cgl3%cAU(pWTP6^ee$F8G&&>bP5!1!~e!5t%hSWdHaey@;(T0Q}oYVP#C zR;pD4`B`O-^fkoC+#xvIGVAF;EZ;5{#S_$z7 z(0%o;axE88zHs5|rGLwR0K|y@XBZU!7i9nfJ9hzm91N5yb7>d9GcN<9D3H|We3k1L z|8hqGj=~Y6#%GV^fh3d!Zc|xp$3~7W2Ef-xublIR-d|MwE9nU!R{?Aks-4(&q!?$l z`|!pJGUU)xBeHU$(mr$Q7rThyJ{S-!LOl5aQZS`2AP|jK;AT|Q+ylCULhz8f;mL~I zg0BM}p%dCp)C8J(S?7FG3Cc%{pX~5oiBd=EXzGgL*~e58B)K^9*sd{{s@Z!vabD10EAV-?KP)gf(MX4ofDO)+WuiM^WD{OqYaZ$ z-_z-0gS9~s>Vd$p2x|a5#3wOSWmPycHmjDLCMQx)lq6@%4G#cTL??jnGgrHe?-z1x zTM&!~=bb762@Te~}!@l`C1w z!P_~xaFP=0AhqC?O{8$;t@$dfO&)JBfXP41qvhR-EgYXXF~X;4s;+5|f-J|W&~afA zH$z-px?&3_71wEPYF>>v0Yw^2=jb(P0mzHIL2%=j`}^bc->FZ9zS%|{EmpP>yHo8cP` z(P67v$huphb^r)^eT;%z?~C}s;LOHZPl#mS6@X51;exQqvuVLYpfMSa>fx#|3oB@QP4tB!)X+L#b?e?J>)8iubM6XoqQ5d zWqKw!V3BmSf2FmofSRS+WxHPZ9f{>urn6;X=LwK1yS8~w|Dq%Ohg}!A_$5Y?dLYP8 zPwOBgFYWo)Pt*#}^Gnd2>dQnd>?KyLyZF3OL$hvh9B;AI>{~0wm|ZI9v??7y2+bYE zw_CozbK}8Zz#+eVHds}b?{o#0rq*Ga=uvP>Nw!gPmQec^R|neKmPISe&OEjmvqsC_Ee#mf2G_oD zJ(q8u&wqXISy-jXpa4@p5h@ZqR%W#w$G$iqxcp^Isl^(=-5c{7Ck#8gWOGoemM*H4%h>}NM|bekx!_0l+q#I$O=JkxheO0d*s9J~ zPWbEJN$&p}v)KPfS$qEtHi!6A=WYwFjY+lBdiQzi_(2poyE-I_ls54Na@!i29e9O8 zjH-cCucise>;hCkQ-B{7MuJ?nKxW^2=P80o*&aLuN@0DV=iAzE38KUsEDwl2nsPiGG(Zipb_mkg{ZZJN8bD4rPEdVmw+0h{CNau8;ZdrS$)QhTX^&0bFMxXUmR9r?q32F*J=CbaHmWb6%J!P!=D1PLFcgo|j^&wisHr1j;l%zWQpn!O>1@VOP!=~8BFv}bCrs~6kVK9fsq=8VeG5N{Z+naD zda1V#&rQ3$J@lh0s<(Kjc1uBt}j)oVAO(S<{|(Q4@0FH z(mP=03y=WyG~ZZKy zRuK_Vt8k(o2Ljcq5ymyAhh-gg5TUNN?s!2_4ljU?*MYZd=m(xqO+gmXoiC_{;+AhY zbE18CsT zJ$wgF7zWA}G~jDp_#pwMVm59tM?AY`vRem?Xp=zf8(=h-BJIdNb~Mx%ulUA#*vMMpGHv!_c^}?I@9E; zDR3rv)%&PUD~TNp7ET7|!5mKT$+T;!X_eb+sEij#cB&O}USs?EAWJ<+ZY+c*>errw*Tu(Ao~}fs+Y$ zU1i)ZU=$& z_S&HV+2qRW5k*sljrxrm~k~c|a0h58xGK#wwowt@PcycM!l` z*p9`c2@o(JOl-f-JcfidMDGt=T-0KXX7cJIid(jOTPrG5g+XbR4qX3xkVr@Yo;d#m$loVV-L5$Ma5;hpV7Ir~ zSjXU*?EcV$Ey~6%AQpJUp$r5R_d1nBPR!?`@zq9#6>Ej&1M~9Z`CC9INU)s8-eSjT zf0H)bG~Xax-8g%*LkP0l}0LM_6thLNBA;>FEk^Z+GQq0loIhkJjTtjK3QA+ zwnjB~vMxhH2S}xdO#r6K>?XdpDygg5#U}$kUKzU@ZC4?DYMSIGan7Ueu7y|8%G1WE zYG%&rza1eTu?n{dl{$Ayzbbg`(tz=L&&ISMf}Vba6euy`Q9sO`Tf0H4>JSGQX$!>y zpoCfzyG)C9T8LMtmdjl$JLf{KT6De1%{T?IhyvB(hA#7YBO`;^XOeCA~^)D-)T^91hDh6S_Uw zB15yZxaXm^HHi_^SIWoHg{RR)v+feTL5Jn~hi@BCS86{zPuGE>^YYW1-M7c4b{G6y z(leb`M;(BQZv0IFQaeN^tAYhjO-%Uvrs~10=%%+e24j+EK|NJh7*7#iJ)bqd7kW(F z)Gd-szOP>2b%B_~a0GxAXHuFKj(fHEvsfmnmoUkDgMfuy3XUlm$JBZ1a^|Adt2RpP( zOg@+z26eOP^>jSuekcw%`?1Y^C;*Wq2o^>~Td@cX6V&Dsdr7!#z2W4%%Q3Jjd791b zCfBOu)KfdL`|&x&`HdFLl>e}J5mj;&JbpC`nri$ZfYyfd7b{)`w-Uqj@xc$ z0wDBs38EFXt^o}h)HV=@3Nd)@6#1SQzgy_i@7;oX2y-tlb;j@`1X$ft{9=l){o2W( z99P)a-(O7TXCV?A;4v3sx3M&#=CXu*HkECp4(@u|FKubQ)33)!;e!m5zo`8%t2#2pQ`-2cI0PD zjvmWB;M`F&+;~OXJHtw0M zt@HaOjQ*!Tw03EjfT6y${k2p~Z|}gFSmH=7NBuez%vZK(_=-pV=jZ?9%L2C5IGuOI2(m|+E~`a{@Q$xjK5zIa}&4;!DDLT zu#vG633#ed^YXL(%*|zW`dYzacrEX+&0cpk(vB8nl@l`xbRKEWB|MkCKDL}?!Pj`D zl0y&`gYNDw&>98ww#8CwC?hb+{J3C2~%xF82x#EpCWEY6zqr>YW9%~f0)**L zbe(^cTnGLMz@-(8&+AJ-nm9xti$4xPk4A6~}OgD|0_E(kRH6BQ-HI$(QSQ zeZF)cKylhvVf6H(J>I}h<>h;4U*AJ`v|c|;Ozg+EKLh&Y86!^m-E=bCA$-(_3f$bE ztrwuOxN#sXod`>z1(2zH>pPy zcbshJTxHR&&s=1OT-8>o)lZFl+anZbSrvESJMVGmHa-WnzrApR^@dDk)H#3 z#L|ADwoa1{dY#8TW+#zKGjSbE3t_*WtRAydk@;K6 z*Y!^6&B{NO6(Un5;o=X5m^3cD$W)4(9#AXLnv05QS}I4LVe1d3NFr-t}f<-ql>6p_~vFcwa|vz-8ZA_hr5S@ImU{(T)GMkQDRUi zk2K#L*uU(T3_ZyHEkijpIz?eVAGm9N4)6p;KV}s#z81~%<>z+Z1Jv;u#}2w}t6Yrg zalf$dZW(i;D#^4)Wb2&8ZZj&tonO}pZ+$5uZDi`u~152T#lOpv<}^jTMh(#7mUzS+pYS8gR|LMmX;lZOL3~6ZEgJNxKK0D4`a@A~{<*_gBvKF)0p+bg4 zgih*WnYcbCYL8JE9=0k+--fgPI#Nl0&9xxnN5gksfvAy>DnKc`kMFgwuJSQkeWIwB zKOh@X(%>_q$P*y`%1vMTCDSR~I_!F>rKePG}9!~UPRHhhbhhFTdyLNU-35rvvBCAX?H!7xNey|^HGfcIrdHxOm z<*)(ho6CUS`$^2@O4k*i9L}cPoLib^A&bkDMCPu!2MzWW4J3%Wp>_9cFxA}0U6jxa z;L5J^r**7;?$D?{0n}IJ$Drwoq5z<0-H9_Ia@bK-pjdj6Mk{NUO=%Df6u_UpbmL}W zKbOe@p|>uuhkz#=pi{pK#1U>PxCO6GO{Wu*sqOW%6St{Q_3d~`uSrhL!!S4NA4Jyp zaBQ!)vu#U^1A=$@ZSg+M-rF+1VoEL@Tfm^d^VV}~#M@WAYjSd9irHChx0ZMInAb1V z8IvGvSFA2uMxLpZ@YaTub{b)I|J7~^7 z{+&|q?WMKK9ukJFGD(|X>Wk}2WsPja=diA)AwTd9g@LZ)2S{87@vN}p_C{B*{_t*Bp+VRCOKxFXp(wip=?)rA z*3{x%=Mbt`>XF`Wt#gEIh=%tdQRRY*m|yo+`k&tQ!(;)DeoWgHSOzPWf7Ml!sqNBPfewHKjSfhpcrdM+az>SMEl@)R? zbC^jZ9S6KwXz@E&FFbHEAGLH4TnYUyj@U)fj4+=p*IhYw1;Z?dQ)wc_IV5X4ZLy?Y zZxoD0iLG_yenpi#Y}vK4|3j_XxMIk8Zz{I~0_@CN^$t%kp{fdfTOr)J3n0EfCw;4inLQSZp=%IZx_dIKh zWrY~{bnm)W87_3|`e2O*8z;?v8`Ypez0^Rwf^e|)>8TMSYc#;$o>*PEJ&)<9;Lv$f z&Ip-9i18Os|M`-b^!hcDQff`eld{sOV9y@X5<%Xt!;{8=Rw-$=izS<#_B|M!3c9T< zB`=I5X^sTN5Gfg7NGRQ1X}oWhRb;+0youQT71l12zIKB>#77Qa>mBAhV??%}Dnqh( zLJgZcOQEQpQD1{zj-KEZAUMGE^LR257o875V#cs6T0NLiI+meWj&!~08qRd;`S(ylz7Kb%a ztb0o?IQ5&dqL{Gf zS6H11G2kpdGLB6hXvO-eK=CnK9)KNGzIy)z>|dO|?Nxi+lgDYS0d&8eI$Ra4=1M?{ zEBsPiPrbtWp>KfusT9+Qx#D=P(M#GfcjKZzH2L%8S`S019=cERA2Vv6da~+)^dyfN zN7@;c^0m1I+RX4#gSeugS0&jxXwu1xMpF?BxQ3%I6AN{Y5jUHBMOS2Gfy=_O==Fb3 z`OaTcCf7Mm%8toMxfV{$>RP-jp;b1XX(+UG8L{J4xu$K6EmNq4CxS zqS`PqHZ2Nt^V398j4Q1olP_XqJ2Q$XSZE%hTe(QGaA53O^DE}4(9wCWH~SXQ`Pcpj zz4+gN+B_rIp~b~Jfa)q5Pom<~s?OG{QTLvIxr_fV=d)SsKw_qDQs5iC)DFCAkg>hF z!dLj~*Dr58`U;f65EW-w4htZ^x9gh|Xg{8sN;_5>XG#W?{MxS@>cXY7<~;)inI1_W zZ|kKl|9Xw4O`c-jFTFeDgR9x?$hhN!$r8bf=1mFdF&kTFhVVKOkZv|n;~3zyV`Npv4_OH9m4V5FYn>{^(f#+rzorQvP+H zbF4^#+Lj@_J`emMPrfpBx&~V_bvor!vhDzGL7m8yIH3kPuIvk=0A%76y>Nzdn95N| z+Br2z@?0Lw1_p{3Q}|e45r4k|Ty$04!PkH@CEdat}^@VL`KV1zZMU~M~P@6siTEg$4TirakU0u<%7 z-^h4;2!!PGp+4>Wor^1riyKEVB38cINJC@qizbYK2Z1{>hzn=qrdI6!z-M@YhX{{~ z_?3RaRrZ`A)+>K_nBAf_67i~+vf*3wfK{H=`%T~>Ijz0> z1GfPzHjXAB@B7xgL zpe_`mO-!55PW;C5HA~P@LE9nb6q4V!&hs3gg^q9GBtAfGz=h{epS-byIc^RFmn?K2 zc=${`dhFo`VvE)F{>+#3q)x@3rEXoTzRHfaI~6oK7G8`CTj z6CFLqm)!MV!l|IkBX>d_9<^jhdU}m5cnd6lj9y!*y6q-|6qUMahXCvZ1>VYDp66l-**%N_ne56@NN+r z>x@Hw=jLC5F%)i4?OfM1ZVB#w@3xk!y?|ys&4R@8c;S6bhl8(izKjNZ&r&Z!52ABy zJp>$e_FObbHJ#UpFJG_U5Ib3Ue0Cb9FS!<#jojE5&}cj4QYYJY!u@<;6Cj<+^81RvYvl7tD_(XZ?SyPa!G z-wzvZdpq$+twI7sY_k%rDt=6^wu$cG-AgGVP_=Cqvf7Q&o>IEYQ!tvgzgW)cQaTIRV!%T)uw=L#d^E zDYet|^ZFIhvnYM)Hz;8P24$_f3QfxloQ9-~}{I#P6Ji}*eXUEdgjTb$zqM)bR zg13a8nmhOXXb|fYPJ7ir?|>8B@QFGWG4%eJs6o>s&~2teTXev8$(%%iKT2TOY}`P? z{7=I>--EBi6mclKMP8OJnvk6v#U$_*@5HN-=4l@UagP{jqy;we_t+R-KNrDjULvjC z-^!Kfa%C`cFP3!8elH6`*$^`TSIL;xFVm|uNi2mK9N^!xb-vB=3MnAhmSEqrcNo5&b5Ey4pGHv@hc|W#Z*~>GE8%~1BxDvXW=(Pz>vgxVo zr@b66*?R58OC5E`?yoV$_Sd6b`d4I$=8O&i9AVo|#)g$3srZHycqV%>wlt-!tynUc z5L^f~?$M8Wv{#`t!hMvMw-PQ=6ek0ba)p|3IzD0*Pg=&Nc61hfSm&AX)i6D4B0eggG~q9Hm$=Vj2LaLYEPwsb$G;K|{gba) z${%|1-xR%1{ps|eIils+eP4eBi={uGKC5B$&7eOl!?BKpo2q_sec&uET+W!N(f)uxWdLmy?bkT-{*Ht3# zBDFEeV}3F8;&s}{!Sr2b>J>_|Ke~cBNI=E_79!E*>+0*EIe++AW9p?zECH!!Zu0Th zAb~iaNRLt2#X{&=2XG&CKUKdx%>sd=(lm@DnEogw{s!#`v&NbFp7=}c5O~asPlE59 zPj=G_R>pZ_^BQ@YAp~~{*h5f74NP!HGH(W!J4%b@MD_7LtrnEZG8mJox|2!UPsMM6 zy5xdn!`7ef-#RN>9kl)&D`D&)$Pea7TCF%1l5aEutsn?G?>8>j8?n*lC^47|^*4Ou z>J%ow*$Q-aLp716zL?oA>hYpx8C#?IdO~bOC|yx`D+i82qoY_aNGWecR)KkjJ&DCo^myGrnu|V-i!3dv$wbbktfrr`vy7B%G z5TizGBIr7{xO4YIVO?^YfMLb+#(h6#*mU}LPo4_|B-UGvan~HZnO3sP;;u0X$miD? zu_&<60eA6aaaTu93kJetb|K9R2Ih5gn@s_uy{~Z3`5jp;^|3^@lHA zw{xv_eptm?=6Rq0mW-@Wh?OhQxyUlcJ`{VqLMTr`Pa`S%o*CGLm(T9tW)rX0(>~LV z&O5EaaQ+1%@C?v(5yY4I~R(-Nw$M zJ%8A=OCpoeo~S(QD|LSv#dz4j93Z*!!{t6Kr3E zWsFKv$IhkDHL;O^arHvSlsvbG$QO=hla6c3h#e=1D=*J53sLSc7b`g%XR3;k>yjlt zPg~&qx?8$C*d~>0?^mI;#0jRKp=x|AzNIK^4i4t1HMcv+XB+YzDU1D3y}13DTX?K5 zj#A$gYtBI?Sro=eCfOwC&V{QKV4)3lou=^t$!h6(Pr4I)NbIf9!6n?F8R46v#Cce^ zjF|0-Td`1E-hEGP==-dsvl95gluK}_p+0c7ulB`OSWQs^jF{h{jq2@{Q1Q*LXGNiY zt7XiV{#x9ZTYA4GH*9-Br1GBd0zi_N+?1bq7hR9NoS@B<`{Tg2UxTHnmB_we`P?(e^ZhEB@y`S>RvX*SRE|6xT5)eo%8OF&; zYp=4G)>^~nMFqo=5kyrXh_+9hw|KfUdypM7jw9F8?<8Gc9lbOU6JYh{oqNiNu+i$=LZakKO#ez}>||BnDK^7wCd{l0=xNx|6X#o+Q<|9}ptx zaAMlA!9eg=nvrESrTtH9%%+_9PXAEy>`fFpq>xEox|y}iaEpp&PRz8Wcp@g0_BIt) zJY%1ZGcz6Cx@Q-&SjBli2S~^`yR}C>?N3MT5)OWHmn8(zoR{ ziBlwR>oMn_^9`(NDe5p;f%V1(iJ;$0t@36HUEN@BiwkBZv0Dy|yEu3gQ{O#u3Ky_` zy4M7F(SzIzmL+^+{O>k*!MY*M`a`x#O}7q`ZwA1r0Ug2fgm|Hu=b2)p__}G&1kBkq z4Rl+mD4gVsGlxA=(~nQZF+@&0%$lI5L)oLxTlUh#@|SkI(PpySy5VX-pVcx^Cx7hM zN#n>Ev1|M?Jxr)9xNPKbQ9x?NHQD$9+KyszQmq{q15pWex^TH`&THY=E=QpNt(g5} zxhMhaEr)k2JLao1ne;mLA+SemtQ_54coSTEKwQQmPz5WX=9AQIkhc2u+stdW7V69UH#3|v3x zR=M4xYXUvS&T=SLfzASkx>D8HfxNmd17FrpEpPWRlJUmF;SFzv75f2Yz~lSklL?u9 z40TZ=4ZE?hJE*R1XYEtb`yzvAXS1h)1=KL}IFjKqFfDBeDtBUo=M_kZ4Ejpfu3O++ zetoEaNuenr4}%V^w%7w-_$jl1oD+uy*TZ*Vq^D4w1MpvIL# zI;C40CMK1S%u}rcJ)J(?n6XIJ9IFGRZP2L70ZEbvAKAf$lpo9k&UWfDc^cVZp&OFI z#u{vN#b_0?tiImW2LeWo$e+B!`s1IQH+r+GzTB^pSY3{@YM?eGp$V9_?XSCm%dr!S zn?n%iK|%b1W{zFoRO!MzSB8tKjv)@@xc9HPSB-*>A+0_s&V=TUFJ8ZTx5gyn&;eYe zrBj|bGt12L99(4opz0KG``_EM=LD&34qWd}AF>R-vMRF+p@rsaRi zXRW79C#y4eE$3uPKk}(~ZE*J@?+dt#(*78oubh2eVrT1}Pd8EoAB^GtKDF}HE0qRU zHTU&HRMUWv_XwXqcma8P&;H&ZNct?t;Lo@r@iTc*9Bo5Y->z(YL|+-7#g)sST#E z-+#N*L9x0&n1-WK#n%Kp@K#_|`;FV>N^TYd>?);wVYr%+rgAi41o9ns% z7yrHg9=*8T=<2vro7He#Yk9qW(5$Uk2yIk)!s{oG3eFL#hZQqFqJGaNTiZ>9v>GOg ztLkoK8_zCTpxLtb$3KDEoIO^Pmw5<(1;(RV|MbjQ)7tRBTzBGG1ruDd$?lP-a;N1# z%$fWO^7q?yu|^u);we$6rdVN#x%;Akvz-m(_3&6<&W}KE{ol({;IJ9|u+eRH2gTI! z#ARlOIh{EIlf{QeI>~kHHWYsSSwDh{Rc9G)YBUz`_GO}Q?X&D>bAma+_+->&b-Lr< z?|PRXROT4(_(-)(>_7QEH_vW`JAzaBKN;~uDBY0)7#036cLP(^LDLFx-EXHH2O*u@ zv}-}0ZyU>G9%?7P0s3O7$^yqI`y=~x&)=I@g;qriQ(UtTeBRtyMuyEFDNHwHEb544 z`%ji3LNG*zk43xXTjXj4=YRRm=d!rv$N%zc|LHG>4%{?qomtecsYh}z;WY|jf$;us z9D7MiU31nfW5^^!O}6Y14;kXm;FNY(ym*ht(IG97p!t^H+T0Txfi!?!Hi8zClt}!& z$8)cNCrikTwYn-g+VtrEe%5NdrWr={DCcD^I?rjFg2Vb5Xmw(0Qus4f_0tQ^dNZCL{Um#6M)Sgq@S zJo>;ig|=;}&CTo(|8foPpK2#CM;s`RsQs72#6|s=$Lb{gKMbEMRD_hl(GBP10;JZe zn2-^GKo2|6i!67?SNDFV#5QH~>)E;Lzgt|ibDG&1@!iRNp%XP=+9Q}2brZ9=xEX#b z*qxi>&k{7^aixl8E4?nr8N0Oi#2Qh(wz`$ht}hpouxK}4Br#=YcNmdUcibY6DFCGB z^grxJX7A6Xl8FpXQLa;4I=etC1Y~1Hh;!Q}NL5DuRD(~Af5s>cZsr{*(Db9s8t1J% z-Dp(o^ptm6##m@)SX3u=-k{Xsxy>tQwFj9b2jF+#u!V8ei>balgUcv~_59cIW5B9R z?0M~=e0OeZm09GE6+vyJT}97VB=_4Sx@(BpGgQyT-dWm}y>_RC$7xSmSujoi!7|Dw z@7-$7zfw$I`ZA<{NZ!T@SvfFvM=w|(jd+k?Z(l2M+opFq+HF;go@r5vT_#!WCtvLv z8M;g3DSi^R8W473Flq@heFEioJTvau&3e+$DC^k!`Q}>Go@C$WNflTnmAU|gN61c3 zw3c@VWW9|J@!jD|FR5CHM^(C5;V=Tj=}R(Qb(z6kVQ_ZZ-Xd+_PHOjdU5)?% z?sQNlTyfTR{K`O0J&BvEC@FnD|69`@-3`>0P`cc7-aj3u<=9VpK{Wo+bIt^n`J#?T zjGNcOP51WZ%V@_+EM-N4i5C>z~$T-SyiTF9Ul!jF*>lTMbMa86&LcvsMYoU+tUDYF;9`x z@fH*EdP@N23L~QiYN1tR<>bc|j!DK0Zvo$~QP0KnqT^(iR&+orrz{zi&OyUCO_zR$o#2YIn4ZT;Amx_siA*yljGlH@6I`+QBqq;w~t_;^}5jn?r_ z5d6sS|6}eu!p%5K4dmq1+E$`#e|nIsfi+p8NMBd}WR?$9Uf{$9yNzGoY|0M^KE7 zfuh8UAw8-MA-cjRG_R{2|Ger}7vCCC*Dr@_aJ?k+iq1t>rJ_Y+=n;G%lxkC6J7cKH zXbZ_93Y6Pe5F4;{oulnNk)|7V>wK-<71EeFO6;1St7om(`@T*TXT?gdeE_zYSz)2r zHT6o2?P{aVOCRqh7uLj1pFyV5=KQOi#dTI3D6}av_ZE&VpwL95*Vm z5<0{YHiRPUyo96gVCz@~C#=CvJ;j+<&NZ{FbFxXcUa7F{UwoajeCKb#e>!gZFN_xj zIdd10vpp6ndJy(mMU@9f6Sq3iHQ z-zR=n!w#8mNk>*+0FF{{gAcW|Lb~a>J$+{Sjf!VLg7NJ6v>j!?w@F`>RDMeY|g-c>zJTtsGRAoco()O(Ve=1Ikqb%FEZ|tT45Q z*5pclp7Q{%|87w|B_R)-v^#c2Q|XXvI* zlt5iG?touCKQ2TDDLk0b_`8$k=ls$+>EUxeeofOSqvFiC7WCh0BX!G zjme8T{NkD?tvmiTavqkaKBwu6m7d z2wSd2U&2klOG>_xfDkPH0i_l-Tfa+le61 zZa6zGO0DaPM+H{i5f_dUd(+3C{?x}?a|-fRT=QXPsf|3#*xls1Yn{6)V}3gL=E$Zn-9@)Iz`bJy6{MCWltY4cSo~9+JU)?5ho(bH#78ubnp$C2~Ibp z7n~! zd8!P`q0dftK4pIiIYc0^Mk+mRx{5#hakGE-t6wSsBtE;> zRp}iNw7kZw*eq0~o;={OX>Fwa{0oA6PRJo!LH{J}3v%~l`prr}9QBV1D@VtujwUuf ze>*mVONhV3j4}jYD1T<|eg{_S5f|r`62?BE%cesvNOZ;LtsI5oPNpa1UB3>~^d*87 ztnSDG3v!X5&C!!@e7rfpJsNV@GUBFEJfL4XWs`7Q-sT0f{iTkc@+o)7h-y|HpG*m6*W1Yrmvuv9UO0>(=RC8ljeUN=_(&i$&noqkm1i-|F$ob z?;E=&yH9V6Q>lMj1mf*>9uY`OsGRsyPHgP~GCJXFg+1l2(RZ$H3j@tG9iJTta0(qI z5?J>(=vL#1STYF|R63_A-&AE6a)N+$E}gBR|A|SvQXa&0)mN|dXh{4vhsbf>YaoH4|-ZzPRoWZf%$}sVqHSFfjP75oj93qfhjfQ z{ByLcW&yQJiCG7yADZ2KD60Rb>RaX|$%mrTXRwog{Af`T&R>g0bnTtZzGWmRzG`yt zmP%nWo2V~JhI2z$@!|N1;I1XX*5-GbJ69}#3bu(Qn}lzl+rLH~xtwB(RMp$$z^Sa% z-{rbGho`r0qoxh7%sgvVahmcMBIv$wah$Wx{qnclm8Q?P#XDLRS9a1_WTv1hTPsB^ z8#^|m1K6$G&=Pj_?K*TVaZrE}3~PK9whCBpprZ@lz;jvmd1Js|zZ)LX8n7ml5aB`JH<;Mafg_`fFfKOPYvU(Ffa`sPG7(Bi}u z1T=unWT*Kmas}F8h}AhJ-UA#zgSH$pLwG%EQdE8X1j8}Y8Iz#k2ntP{sAQ#g0q;`oC{h~oB#-a=QeC+ zqR>5K1$@@ThnHsqk}^F&)3CY5;VHo@>*ICAZssPI`*Bus&((6fpop+bbJ}vh5x$!M zNt3OA?o^q}7(_P3Y*2b@u_trlm7isE%-_`2r4=XC6I5knTZCedq~JI1LUZCDP~hln zKz4vfmA2~yDgT~MozHzn@cY&+TROZ95}c4{-cKj z`9Am)s^uU{FZu_a+xTp!r5J2qo69e$1N&OknXa`jjIwf{J(c5|n0YFAbs|_370OI$k zj=zj}H6zjuf?nMn_n@otNiw?_wPXe(Hk(|sEeh@vKb^(SbyxH|X_=??=DQm6@hY!W z44J;Y6gXbkq$R4&Ql9l_{6N6 z)C;i?PbZT0v_hp4`ds6UG1m`f#cCWMq=D7<42WxwX5OY+!aMitzt*w)D=+f$AyO^c z?Sk)fsEV%yqfNBsNBw3L4VR?T5@H4l-}sEE$|GO?>8+lq$5x|i+`Aw>{y5U*Z+`@* zFoBTH1^h+)7X%BG;ZMnY&mcQYtrcEZnc`(NO#+IVDp;J{bQk*>HTMZydmqo(dIW7a zOy%awfc*cq)cx{f!T;*@J@M=Bd#oSn8fC$5V2ri!mmD$>)lXK}6i57LS9epUzX*#G zBL`IXaew`+#fugH%GyeQuix7?m~S> z#K3l61i=qC4AJzjb2W>~aarE9;V1r*q@XG(*k?*1CBThmOb<7!H>CjVHmPHhdi@ps zeqp)3l3#8dJFF@iR7y?YAj9{{3Y5oVcT)oXFc02qSrpr6L~T?($My_+VC*|5Ew+^6 zG7361Eqy?MP~O6$T$iPf%D$iccCVwomrh~geDjwc1+os48H?z}N5Im-MyaQbcv=F{ z>uUs`qFX|`yjPww0@<)YK`pHN!(!4XGw@ICnJfY2E|L>pMY_x;=_~UQJkZw0btttz zh6Rf+^DgCE-?Ik??Ht*k=ok@B;CyU$!0>_8fhUeRs9dT(v#YItV5>T~nuOW!XE7S=njh;9 z3>s74!@MVgp#rm{eNuM4aVKU^RJyhlv2UKehbay2qW``3;4_~$)(^V;YD@+KV${TL zsdQb$w!GOZEuEmyW~6JMJpU3Rb8$cNI)}TP=0rLLt?WJ1FNb1dmQW~?1wzolcoiS^ zz&>FfUTkoKZ&vNzNYqZC95PiaKn*yezAQF%c$C4dE*uq`sj`FyDf^;r5r{4k81G!D zQ(D&TRZ|buZz*(;+M>bK`2EaE_yb4t@SHDLLvJIIO4?s&nf&n7I!$aX%Xvm#_%$iM z7cmQX8i?Ul?$vb8Az!8*#j2C+J*rM|K!OBOB~T3fDpHy?b3SFR-qF81pIW07uoF$F z-#2EXKby7;LX2{ZJ#>{}lkEh_JOV{RUh`o+mU^pHD)0y(tFHdlakaxqRRaPW#bbX*Y*jbpjw(YG}-mlyV z4=?d1RlyKM1e>7RJj!>$$2z>G+s_=>Js8etb$&$YF;rimzmH?3YkHdp&+k-L?wIKK zEPqf`E2TYJ>1s>u{LLt~NK$>L@fyjUuT&QXtIC=q_||==`njeC8yQJq$0|z^u8J=c zl73kKVev0nVXtAZPX&e@Px|Mb8}63j2*-%xV~CNm#_4>y>PAPh)gyo>-iZ)^Zo@`$Q9%y ztd|wS=c$*c0$^^zKKivy5&QQjzUHZ zcW&Uw8$IWa!LBwaA+T|2YAA~FHEFUn8A~f%j1~f94k9cgZ)RPYW@d{$(3j_t(&wE< z$vd01G7}yH`nRs_vS7#AlW0eLp3QezS>Y%(APCIrv(wpB?Cua%2LTAP>xGW9|K z3lV?&XCkW#i=(9Y@D4!jpf-D0h`z6)xz*}j$SuQi0DxVJmckQQrM zYf6Mu!q}IVlxA2&IJ8!FBwM{^N4KcGeN>)o0uMfkQk#vpuJkn3#2YaC&TtoT?WZjE z)Q69+Rw@U@$tG&AAa;}W>Q{aSAY=5enV<0%oewGP9_?(_QF;I3sW340Fsg&9Pq<{* zU-4XbtYE=@+q!Q_PMD+w@nT=Z;cnY2t2OHPSi=j#xu1^%b1tCC!Crr4aW*T7%kD9- z&n~dlcYrzt9L1<4a&;$4Al4G*2nIysMYDN;gZCV@5^SbUr*ilP0 zj?3`f8LD|CJFaZC!)Fq41*JcxC73VU}F0CgY~Lm)hiACl4kwp zz#79L5sS-(cU`16_me8qq5_3u1HV3+3hvrSnGI#hPGSkoiP^VBy|)J`(*-^ zWyD9=e&6Mv0PZII&M%!K!M1D5W!}_I-)^@XaqHU*OQH)beK>xC!&^{sYCpvXpWnpd zUq4jT!Z+USi!mP&+L z^c5ti7(ba6un4Dre{++nzK^pUFY}}UHxv11s{c9n_;9`fl3>r#_v6vi4C|3Tst`Jp z%a&X`w`$Q)J@L2fc&hh7I(06-_H`n?zEeQ?zWPq8UFX!yqED{$jK{ctQ7?d$D#yAl z%SOF@1Rh?pHJ5<^?G`?QN|OgDOGn@A>o8=@Iwd>-?zH)Y6xHbipeMJ{(w|=N=k&cp zBXy1!zb>Fl%$~S{-v!dq1oz90zI;t;HZ_g1w~#N!q8Y^|s7#3woBO`s4i*haFx)Fo zB*O(MKj4o|Lc%UKcrT1G_+EW#rpt2lb^{V9UR(P5)a>uI*lw=vuMY+8Qoi=3=qu}N zVD^Rt)OLe|oc<2ZRDu~aHT=`*B9?L}8eEV=7NP`KbZC<~?$WOtTi0J?C0rE_q{->+ z{%(j;0di`#5lpgEfMHZ4$3+WXpP5HoMXLKdrh-45U=_sp?J!aI$gP{+tOCl@G#Ml} zan~Aq&`7l_hXUeVWf=V5_LVv`Q?clg{MZ^#m6)@tvf; zIsCfDuG7oZo0Au>%3!Tp6sRQvtqOdxmL}Rd8(#a5F||`Uc9qk|Iu#h+$;HIU6d7Pc z|HDU;!Gnw_xLUr##J`)crf(mzLI|)7FtI(Z>s?<^Zoz}k7R{(gLaPdQFv zl~Gw|P#ed^i$A9nSkl)f3)xHqtACJ$k-U3LoAj{~4vxXQtD6>k<9U4Wy%P9b=Refu zruLys6Lkd{>`R91@0DCH48kpBl~8lgIROriw!G+SX!=biI`D0t93)@Ou-He+KEp<> zgSUW9(Q{uD5+_1r%MTJkJ$0i@o7nJQrjn{2|%Yx9H`k# znWnCefwbs_9(edTddaW77+?q+{MWu^am$wz-W``?Q&;vw8XYhXLX}50W4jx-eKQ*} zdqrn>HLY(GsMSUAEq`wdbNDAn82^X0loZtDD`3G?iornI@Uw;(?Af zd!OLFj*c;fk^O*#;F$eZY_vr17mk$DwArV5*z>{F* zos|qaIj)`9@A+oZ?4Ln+rT1&(G_9GfR(-UouNiRKTR?Gg-9EquG>m}3%1(A;0MTS!6iQjqrWoeSLoSUT&Wye zeUXn<;T`>6rfyHw{#s=@^hbE^elkCP4D8fyh$r z(0f|e%DsGRcf*ZF`q7K3lg<*PC1{X+B!iIMy}Hx~$V(DR37%c`$`wuZSDk9p(fOTh zAXYkjc2f}yoq&P^Hl)y=lpJ(&13pMrO|^vKwvE(MiY`R(uYN9E%X+I)!Q-l&(U z@9-+YBTlK7wc4MFv-W(U3HbZJofP`3Rp}<&H}t=fVcAsM{qN}DkghND#*6o0aY)!(iT61VI)~RjV1_or(PZTL0Ug@;1z>{<^m{;ZH3{ z;G5qVfVi&oMd^6^0_tP-sF<`B7g*2>MwOp zfonl(n4;lA_xhF(NePk_y9eEA8?i(vec~t!;op<=Kh0(y#!M5;jV%AJsmj8ElJT(OGMYq^U_}d~d(!%m6akczllVc*xB}KGZ*Ds(0guhjT2w+DDkz2uDfPn zH}KFDrvQI2RcR(0!xFQA7POg#`a#MmN&&-c-p;n=?ArR@n|obFa_p`3q$}#P{^6-F z|MMePxMC!5b*qBHooDPOfPFHG!$1+|W48q4h09*cS6R!lX}Rc>wfYlABVA(l`*nuq zV;+wFVe915qM~WJT$$2NurD7zlq<&5Z-Ej1gpGyD+;zE;`>yjUWc4GGfXh%d`Y#8A z9Qt~;mHqNHzbvJ|9H7W|kx>mitpS>8lT~q(>DPsIfH4aEym2Jtim@!IagA-or;_UMVY zS~+sSUUs9gKRf1G!OHigU5q3%iQeEZ=qt!mrgQ8D^=GmDn;}Zyb15 z3G1%`POrtH3{t@;4fYOa;Pz>pbP{GWDs0Cm;CtQ*wG*s{2OhkhP%%#4o8ANZAz#z` zM9_Ite*x*!&OX1(W@t*sOSxQq{>j|D0E@p^QU&zm8$K%kcdCEDJJa+?Uw>yr z*sh-D`>XHX$Mf`DEJR)XzOKFN4ydeXkbUhn5w#)+xq=r!GaI zv0d&rU>6VQz`H`rU$H>-j~wT)g+T8wyMf(Xexpw;SW`y}wsT(8)*4o?(0ppYfFNEV zDIz&Gqv{prfk%pE!&Vuens+z>MGr~G<9b)UaYo z4-h+1fQ^N=)K|`t5_TzDb-&!Qtgg0?F_|%h@UyF8OP=!?_0#}`q zGudoiDin6!gyXDh%5(UOn^)YPun8!WMsN)!4r;_`38$%RO9tj+SFE1U_nM$PpPU*X zcge+`y0O_Moi3nPkvSFo4sC&6L|yk9N|`?KQ-7-mV`qJ|obiTF(bMjETJHORzK&w( zXw??WRv;_?ia&RFBo`%aGNY*CJ|ZZ}Ae0H?F4pfWL$90!@y- zLD>@C!oR=m z>)X>Xx?BWH-JenqZu*h2y=EjKmmGMm0^8Yq@MR8s&-gcN!gPFgli`T^gL)y)iRcki z%zhCAWXsb#mD6HWztO-rSgOD_m#qjcj*oZ@8`(mFNSje3L26E4*Y&_EcidkMwB@Nl zIs@vEMuh!!b$@(H%;FWc1;&Y^6RwwZ- zzN${=vO~AHFKl?yh_G$bFbkxh|KfaprXa%3YVo!4@eSIJC1`k6ru#dbrYpTon^#~E zRp+PjrD&Zf>#8_S-C7)M4-{VH#XSFaU4hq#{Hxp>zN#xw;vW8l_h#Jw(%hI5H>5-W z8xhXwz*^h@0g z9R0XEb^8q}!UV{sgj0TbN4fa~$-67%X{%!A+VRli&mNoSjtr*cw8;11lPARWDv+ZH z*w)Ykm>MK);<{yu&7DBpJI(KG0WLa{2QmSHl)hZ7vV>+ujhko*2LqL5QXao#9X`=Hg(a&$c!|H5^#xc{T(d{+VV5mNQlJX=@)ny_&ft#|{rLY19s$U3l(lAvortp;_e>R;D~sNpzin$Fs! z9Wm^F9ljiu|w zz0NDAeipj>u#@{@SoXy3AZ&b*t!|MLzx^pl4q2S_o8(8nE~u3=hpI)<^PHTKPLvGL zVX$fc7dIj*=MBW!@XECejJ4kUUV{rDELjjZpEg-oq^C0KT`s5;CJT%|x4GQ=1rA{s z5fQ<+V;ya;{gLipmKPqWY6UHuGYP@ZY?~6NSN>^zhhhT5HlLE-dtMuJh7bQ}`07UzMLt zSQ#LNXovlqnCy~V7ve0wwvI<6JIh>Oazz zg;(t%UmOGi-pX9?tZv*5OH{S{7bI0AthY;@tYgGI6FQ-8NdmU>LRa<-w7r|kr)1Mz z7b*;uHEFA5a|oV_^`4Mp)Td90^)FMebiL*|&pi12D0Z9v3Y4ph6Q;=CQ*2Oxkf=2d zj>UpIYSK1MK$U$P-imVvbUjgB6_paraacr%4yVNe%Cl6#r^blSVcNTi2A zZgF;2Ymkki9%>{%o(F?wSD922Y=W+8_>lVqZPonm4032Yb#;*(4MlD1%ngR#7HY*u91(Jic({5jy*zYx06fy{O?cQ@`Jhjt6}xq_ z1;MDdy5)CONKbM7q-%(J0`7M2;04D{V*0~-9LSU0&HVJEzu85BhAuebQQB{-_D{+h zn?$aLO@!+HH>@3H;FMcGZ zTV&OCCpe|co6qxbFouE}c#HbGR)UZL<&rNPM{E{h`N`zh$??+`TThQo2CP0oaKx6I+#ps+qn0Gv^Df9r z_n258Yc^ZGe<7I72Gi7d`O{R?B?b!l@TBeEO|+Ef(zJF2cjC~>#bjx;N%&Xf>}A{K zM6yJnp{h}^P}E7LxML>a`GVIW;zy6F-KQ}AcCQ8xWhvj;_)hWl z8)EkGU`_(YSA|5{+3MaYM`?Ugr9TXot5h)!f$?f^R)x2_yX{`y@qChb$UxVrNnH<})QU5Ysge|&%O3{|h7g$PJ< zI;NO$>bX1O|8%^IH0wF9gaM|NCD+?V$Iy5MgTbW5vKZh3PP$wo8R){i*_Q8Eo+17O z0jS5KrMlXkY!v3U>QEo8n$1oMHIozn<5mZXju+AoIn+OLj`%-+4Pw$iwPD*t{_|OY za)6dXq*M7Pon!+Ig^N+sRynkvC;12=Z~i){wvKEQT#DC%#P+kgXk1dJLGQ&D{ZicX z6q3Hyzv2f`>T~J}6Mf+tdA>Hu&)w1PJPl7;Sgd<5mmEm(ZU&wGHK1_e9kz69iR!zQ zqtpiu=_D6Md*ssjO1YW<97Ca&?JC#M{kOXdioRq}{-(pz-Lme= zin$I9t1cwN+`QyWv!8$zp451ebtx1!?^;zXW4zY8o9c-5N3TCX!@TA?W>+8s(2X%? zG%BZw#aco?GjFfA0AY*q`@ZvDDSXvi*J7JEk?+-j=n3zJcHY>Z+(d(@o?f~=ma$KIB#HNMFZAHs&hd44^{Iw7`-5WN3_(XyMks}SF)4<~&sDd2ZEZ89>`ziN%LdTZi;)Ig}Q!TJEm0gsZO`>Gw6WL zeGx+++wIknalO)Wx`N_%8XLVUJ{Ztx{8JiNeO2xb39)@oEoP+oPy62a;3v4>_qWCe zr?;la#UFfqRW?6zz5c9l2A{XrP!8K#tp%mc>`kk;ZckBsK{ya_#C4Vz5bj@*ve#xV z(-TShmuq5;H8!5d(Wi0URg2@4c{Vb(XUn-BgJ1eNdf#aJzs@mrN&U8oedFg`nJ5gf z>nteUp*Z(W$6k(j1ERGf!%VB~EX&2qdWs$ejA^;`2E-SvWJAz?kp!t4`-2`-buoPQ(X7g8hYlQ(Q^_eO3;F@-Fox{w2Y8}pLl z%9$ve`cS$g(6$@#KxL;3LQ+r(zT1g^Mq{tsUC$k$F-C+PV;&c@e6A#TeH|Q{Va)IhG?=_GNqC4b*<^xZ7%~fhWug-GJwzEBH%15xCLT@2wm3c^xh@jki4)S7o0wIOnm*FkSIe;)bXa?g!~1dPr0EH zy|p2L+m?S&Xk__wZbb0^8gTbMT@i9;H~MR|e$#DR>e-peTliObG0?tN6Hp63nDVAn zx2=STp9}@T=g!wm9u6>&-2sS5v;o7R1YLQ>iLi z_g+ewN_PVXTUYSWd#+>RHuJL_*sI#z6@2*TRy=I|_s#k?7dGXErs=`iuaihHh*}~S zQcD_-a2HtC%zmtqd_SDM5ocscT(9vZhHZw$(nO0+Kt2kD(MZ@sol^wysK`-YTv^bhT0?u)-h{2$hMGau*w(eIq3EUb!P{kK z90i{xI9vSFT%ueJPMd38X1vqz+KbI%qAm{QE~!>(zcnl0+6fg$BsspVYY%)r%5OYL^=~8|9R|j8|OWB)jV~ylb_Qr{}aI&U4 zc7r&*-cb539zM4?WFz54WdjQ=ujgrr-7z(ONso?>lKGQjb{@}}V4omnDf9Dai2sPW zOc%0~LL{QoYOv_vX5JVZotE~_iPw%_Jq>CGbaOtIow2Wqd`!0HL5*|W?HS*X@+_KVxJ z86vzNUf)`8)X%1?jbM(Q-BLU16RLZ z9%`JgEOrU6fAlE{qDy)FOx9}l)EQ)5a=62p&LSy()Nk9UOHeVhb1$YkyWtm|ULPxr zI34corr9hXjTGT&@WW%KDhs9Q7MX=lt5F-!k;lsj{Bqh4N$d!}Ll0FiO^s*i{E4I5 zP9K`SIPr=R1;$pnUG-0?ZF8Yt{q6q|&ie6YCld_p%_WKxoD$;=+U*rhXLo)K62Qz3 zC)`B9Dfd*1dV`ae?g-RuwtF|&D$rTYa>oPh|M`x~hrUC;+I>fX-lA+;BfSGDNXInf zeP}rA?(4HT)wpWGQ}?JPo6tiVRWKXW`8h}uhoUxyJ!m@XD%h%xS9A>b4-SaH{dwpq zeAg9)2>%I@2I36bP;sBkmp!8x4=ls^xTW-sMBr6>x6$eKD15QU?DAaOq~hgM=3_7c z#VlIs*aS7;xYYSU+q$3@j{?%9owAC}&(}&n1W@;NZX33}++Y-kk5upbLu*NUtw+6W zs*GJUk{vJr7(p-qOu^4yy1%^6CAHr~RJMCBjRCO6)^=2i*Se0PP?T#vtmmeBP(#(S z>-F@k1_V7%I=Fj4-LJv*pV1%JKV%f}v&R}6W|Y4W=`C;0WXPE)Y{5B@rIK!I42EF_ zjd8+gg8_g0X<-Q@=CZF(IwlpwapGG+kLE9;5f*Ju$Z$4 zq(i2jJBDq7ZgceM;MWbfVgfw>k79h)lPwP4k9plqK4w3iwYmEcR|pmcVs2!}1*>2c zpTuuGtUNXeivut#Cu#DrqI(E9eWHuBL@6;$wmbg(;F^J z$AZU8TCLRc9wdm`oNu-wIBtFIJ$6uQ!7aCpU;F9QRre3&BhtxMoZ|Ne`s{Hu*)Q9+ zAXWDF%SXIuPo*%QIp$h`pu{ATxo~YM{~%j>$b87u4cyuLt{p>tGE&BsTlBZSzaOyK zVSc52;OLfrnR4t}6-yv)o8B#p**IY4vyjzm-3dPrphC+Y_Vh0WJwoayhK>D-gSrM) z8<@|$)H1`?(+c7}+D_qXvq@PnIlJqV9~d9bio=QfAod11Ni)72d_g>K zM`Dl8vw`D!u%Q!PNBxDAuUxw~So$-{fDe(s{qB6&GLoX?TmC`i)SXe|D4E|%r`%MP z7YL(;{tult`$bE(x^Ao0|8tCx1p6W4_0vF(hwcin(}P*{zf39rhfw!cY43ox<}cPFPLzrsIGcY5e4p z=(C3$Mc$1f=+lY*RGn66RKv!nM69Oe&Q)u2se4lB87m$pgXmgqK1*v|+=o&3B_1EX zjjZ)`reSo|5P?Aa8v1Q*qMpH4CAqF_W$FvNW36~RT)O&z%Vh%nhN0rK?KUte1{k*PvlL{ZhK^C*6a8@6iWgOi+yGGcv&V}-i9SdR(UY4d)kYLKW! z?W)IzT%%YL3SHpJoaB0~cLD3fSgQ?mtZP>&7O02zeH4IBwAD`uhQayW=AST`K)m$S z$ESOqu$>!R&){2~npTuWnJ>Y7YFRlqY+0%7Sl-O*q|X&L<%m)c;Y%y(84%*aUytop z5b9`VeJej)6#U{4Lw_B#(vw8BWGnUBr;Ynyc4JP=TLt<;5kea42Rszmt3s&Uqlwdt zRvMGGlK|Ek&~d;`5e%#YIOb$Rqu0)NGP+h8zU2-bNcPYsOY6G3lyW_`-TgAPN6W^j zw(bH|1sdU|-D)#XH-C>wx00>gto2nT$4!m~Gv@9#usrLZXMe`8S8H6*GWA}9KJ8-z zK&gl6=!axR%04#*2g^>Juvnx-3f|oZXyIe@{@!`aE=Bh}xdmd}w!zBG@Iez9Z@D4{ z(V26daTHy;{&KcfQj_Pa=k8ewR5_4z!_m3i3Tv`dz;)%NTr0=!kcB}f zMbE0*a@-O~*_!)$t)NignpQZMfylEnfu-Fg;t!1o#z2WR@{AV0(B`dEuZ_JN+4Wx4 ztHCl!FgIKbIS=in@rn0jQFSM3`-22lKp8*h+-1Ae!n3Yr#-2Nzby8|kV%1F}nen+Q z*mM8;vxsGYdTzVR+11)%?~tO-nZi$qHN-0 z;k3U4XNQPt^GXP+ovn+8&)#<5C*!ekaKRT?db|gOuWN)OGqWCbCG=B0tw?#Of981j zL!<7iLQz$J?k2bKPM@qYw*Qn7FTnWLxU;u8lDswR;s+5E(zAV)@EYP2aWy?q(SL>) z)Gc}J8=sIKNF?HB{j}1VykU^{JC%JBAZoVH7?dEDKq*4N(}yp_*6%8Qd1&@3jj@8o zbbGR?%MpH5d5plUSjYIR?iFC6BH>>11^bM1-@aG#$dTW&kdHI^K98lwTUU4ne<^>2 zbC7${dMq!?0C089xLpZk0_ zSz%kTOv?(}tu#w9XoHTaU!Ji^^=*F7Ug4qP^T6+k%>o;UFuS+CF;pyTd71;+6{?jX z^y-rCn$V4=jp-1`#d7CYXOUg?@1jS(n(>eogl_%*%0D~LRXwmLs;g4?LP>Qomfplq z`tE&2cc}S~1cRA=kek=4r+b=Rhz zKMQ=mEW*O;*-?E%8Wnh5D+9Sa9%Z`J1%hcPWGFLr6|!KRLOcN0NY#~D`iH z9019s3cq`+DxD@Jk1J-nyt<1+r<1Di8p9!fdYkCn+XPBQ=LKv5XWe*;>5<^>oC$Nd z7%Cyw)~Psl zwITMYTtW9nfTK}9RopL>tQ?&X?>b_)kSHj)@dM5Mr{!D$fE7xfIckk~D%u|fd_Cv$ zG2d?)mW19(x~T{5)Nu|ZCVlj)p-+ZysT2TOGwVaenn&bxQ7_}25RO1qFdU`zJFs>V z`4KQe^e30!F2%wgDpKfTE>y7u^$_twhJ})30`^f+-UkuK_ubZHa*i?XV=e0I-Aj&( z7B}+p>WjYuELiID!;!8a0+=&sfgBU&Aa;!1pmo&m;{cQP_N|HZv?vAt$#8B*{lSXU z^4>nMGgA>-tv3Re817zTXmX zycwi{D|tREeWYt!CxhnpR`MKDVofVs%MdSWogDG-@wisBUj4`Xb0`0@T;zTl{lI8~ z?|1aN-flBkhwJeXn>>d?5kAxv&a&RKPHrd7sV((!m^)Y0j4O+?7>%P4gv=G#wS^x? zPJYul1Z|Ac8@_ZRw7->&hb#beW(7G+ozGE&lI*S z(hEapWukP3wk_Vo1Gweb63Rd$TPwG7=&09&YW4Q&=JE$6G8dkZ68hGtf9_oSe5-A` z6?Dsu@ z7kzO|%Hxq6{e2Arx&b>oqh(eRl&AMI%)25-{tt6+9TrvF?hW5IiYOxrN(u;wl%RAe z(jfv0(jq0@9bW=Cz?Sb7ylUh_6G)D1%K__A{pb2YV}5aC zWMt&!fY81B-B3i1*4xBcg^G~2i0J95djEr2!go$fUmQk>R{R1=wwKSnQw7F`7bY~_ ztv4i8bU7@M$XUJQvb;=@^PncvGpawdk_lXH=Y~pEi<@k9@UF*dMD4GU2nUmwurDrO;SMKg-xiB8~M%-3@(fiBizX zBY&WCJfJgX6MSx~Q}d_FCTGaeT_b8@g_PeGdYTVKO+Ay5(uc7n^AX7Y7_ugh_(RVz z0wn-VC~UT=|M`1SkW;T_y0j0q@q;F~ntII?0bj#B-Itwh+x5`ZK)IFK&hZV7^HMrh z+a8dW8EY8Q?&StM8u-k_uAk!iD}RweXBe?52x2Pj05JAP$|SDW)pAKF4J2%RWdR zbCi{NzE( zX0)N^m^D{vjaPHgaZSoxJBAdWMc!6~9#*#MW%_0{iY|Qgy!C!VA7_{&PJ6eEuks2X zW+v;Z8Vph(e$bibDH)EyG}XL#*d?EH6Ecb4>&?&sfPMD0LO$nU)8w7kO;0t4N+~0K zR(r!ITu_|(F1sFaf!;p}Tg``5@Z@7}#Z$PrhmF`Xv;5}N!TAaIiA#0&h zeCzwt^!b*!o1T;T54_R__6GauIE6H_1PehP-&uoHu9Dp_fv&92NHs^l-}c(Q5Q53P zpM}t1G@^<}?C%kE%+amwNnjG0;=P7{9%O4reE%Fe=XZ*%#edF7^6ztdXHQ@T8I9>| zcm%9t(?LGNKh9PcRF+(fDTiCrD&1}=*sA#2ykSuut9%T$z0%X8Cn9&;XElz7MhZ&> zmMI-#=lD00qJPCFC!n1h(Y4$Cq!i-!^=9rJBaZJEi7GTqayf)FSm`m*aemWME=Vube7@IFCp`t12ig87QLoXh zh0+^uHC0>(F!2gKJxSV=W~(sbF)CuQy#=FW!TeobNypd=iXM_Vxy&Li!|M9ZCEJee z7l-foYcdr1qksIH`&Fv3M^K`FXIH}7zQ%G0fEb8cW|ie(%sp%%UyS$ zZ&3%TlbM=d2I4k1=$lgHSJm`jp;96}-fam&Tt>s^8`#3sf}w8nTKG$eEG#wA;xYmQ z<`D`UINRP+`d6m1KM2rC>@5adfsz!*;;g6G-mjL35Y28Re2R^XacjIn{ItL;jWwF5 zYvp5k?Z7;NhQeeqX|WU1!KT|u;SzHi1^yrit7l1>c2Ui5Kj)5-usdgYvUmv-l-j}S zKnHoBK!@aV8ikZzDMuy^MW8YYI4Goc3Bt^4IWzXX)t=$Ylz_$8HAN?Sr1;Bnf3le}S^P*mh*mC8M}Q*>ME?^BWqw(|Vj5H(DEBUs8~u%w&(tcB zaJ-Sq!LDlA+?QH@12@H<8vf&4P;!ERsO=Z*_QpM6H44hdO>RE-U9G!XTFJ|p!p5F| zo*~E~jh@foC}M8COI(rlUag+BPpA@R{CLIt+*}nn%34q%+i^l zs{q1)FdK)o^odUWiRmb7>+Q8VF~AQT=8N~OndDnJTTls&{%)*$uH6OySD|6u{>wV~ zb9_z)b|7rVG*4_v-O9#A^o%cS`I&l+BUeFhq-^EJk?vTvwF34^nG611N`)5CIzi+e zqO6^=`Bxnl&dZ>b?i#`KI4UW}@;-DP@By&P^VScVJ>38jMr_{8+QH-*0KbfZT51$@ zbmy!_Di6oDLYynz*p_u?vbs{l+%bNheOn+KVFlz4(vN1BtS2Twc|TPHMH72dX$gD4 z4l~a#n0_i6pIBXXp>v5A+hrvy^jBI2X|HSWRCgb!c*+4ha*XUfX-2{h=e_Mv@C7<+ zp-CiP+3tRY*18gt^QRq;X~$@!M|oo>9o$7nrmzO+VQTXM&a*Z6*>hd8Kron&h4RY7 z%BJ}T)|<_ujPqm>^&+-I8)%({2{*r6gZLj z6G9BsOVI_${bfG7jzN@Q-)g%zNDj#(irQpjbe>fRbaO#O5eZS3E%n@&+;n5Ieq)b* z+1kF?Q0%%c0&X+0nFCIVL+8(iNbUC|&6JlW+}73jjn77TF6P6;-n8@8#aEc4zftbU zL3c1V;$Cg&dTgGChs}hRIe?aZM4LWSW_|;Z6iup`;e;1e5{I zl>CNM!gU*`vr_XSqsM+ns4_23(#Y%*o;B)qO}G%Lu!Q}kjTe7TZze68lZ;X z?}bK%GBLzlf61{{?Tt?_)4%|-O7aYyhd~W0_4)Jbc}~HW@EuKtIRA_bIH6vf?fRdv z2VyNSR@YfLTyJ(5cR(BsBQLHP=$roaRRQM?u^he7c9RY5Zv`&%a9C!iP}wI2OK30+ z-9VPpPyRH*V8j}K-!NUA&X~Q;co&FSIc^)lp`Q{R7#=3aSBU6CS7#R;j1m$|n~rw9 z>FC?NWMWit>syCMA?G6?S;ig!%C5l@s(L`2<%qr}EJk#m#?f3dwj-NF^D^JoI0T}$pY36y9oANy=X_4HcCO;h z4#mo>(YLtr!m*5j0XowP4n!(8jh ztErBb@u5lbL!nUQx*rHBQjF?E7Yp{_>hfCSHmSaD&*F-I z4l})KoTbK7FcB!$J$Z|Ywsxu4-?F}7yy_YAGR9i>sX+d4cNpr!LH^DY!MaGd(unX< z#apihSMkpr%L470wEuX4l|bvMSSh_)4w^Vu3bAFmtN>asw-C+s>^$Qy-WpH#EXa z!77Q*C2>OLE|1JEp@;RN61KG(MgFi!9jz7|?NfPZa%NMJ%4fA$kZvv{c4z*(=R+#) zAX{VTvMkkn?UrFs;u6H@N*Nr4PAkc~`--M#xh+$}GMwBu48ibd5Wh=3$y zplO9gLz9~;w%>%HJhoD#T`DYw&}E*I)M`($ea6jR-|e1)d)IQESsoSVaF#Zi&y zexBMW_mm>%u0gS&n`HFsR8^}B$ePVx7hOti9KDS{2^LB+LzFt4j2gAtNSTN@0f2aP zO;9q2R5w*jr^FS|Mc(4@H>3>yj%q#L5CKaTji0&anxx|Hz0ShuSbt^x;rp4iM^ZoJ zq0^ykMt{*VC+kni07#NfyG7B;rNca+U<`@9;a3SHq~3Mg(eK^T{dv9r`f_#t2hSDt z&0VZIq;T|VgXm!W)${(V**6NGt&cxX1x+HQ;%zc7W^*8Fk0Rnmc3_M-?;DH2?%N+n z8^o1Ai%Z-?1Nz$yw;i6<3*BxrXFEgHt1&yz!~A%%ko5X)&Rfg5v_poEYcGA+!ZenJ z+&O1M-fcQuIZ#%hlip!QC}`^q8hRD&_m#U`Aul#?GG=HPn4byK&I3Ar$r$2*zaDRB z3Nm!6&!3aL3}aLAyQ>uVpd27X)pws%T~0u7T)X~xwU*o!pVL{dL09505Z}4wrc}>S$`4-LlLnWq7CHn^ndcEUSw=Gu!;W? zIkUv#M>gE}J<=(r62a?Pk8U}&y44iW^C&HEl_Z?u*0I*x?EFY~&tKJVu0*0QZe`|z z->iAZah0-qBJTg+Da8M0MLTA{gmn>nx{ee!KRwtlaOrIN%q43aRXPc~Z(r8$sfHXs z82gEe`hRO^DT8;(`y1zzD4fe#CWM|nzoLD`x$;}-uaqtASf^dTF|2Zp?z6r1Eq8wJ zwyl)X^qbT*v$zAKp$L8_|7tlX`J}7}?E<~z-bt3LG3a_Bv6!`;g|p-J$a*yG*yJ~M zEB1S}k+Hit6Rt<$#)kSji!~mD4pLTZb}1zae>2+C4^F642!hoCx@7`Y?S!%sSv&s` zrlu28yi<9wqCMKvX|=96;hQ2hfD)YP{a6;X^T48$wJh82`OV}v;wZ+vpUzuD)k*&poT+r!Kg8q3WZtQ*7HYh6fAC{^mKC#ACn4k@wWLN+eU0bFp zG{**zDLmAp+Uw#!F51MpjEU-l<01S z&_FkgYPnK72}QEO#By1%ziIXiW!94o=H?&X^(MzEq%rU2{7Y@wK(Kt|FThLdVc@OT zqIVNAg%-nov~d8HSrBCP2Z{RpZk?LZ)^@j&K!Yf-3z(e2Zv8=Y;m-^qwXtRru{B-)hk@8$rHEinCgX z#$yy!%|N->H{Hwlz7*fP$gZf@w}_BR3&$>ui-OZ#U2uURckr&O->O2H;$Q*`BSSTg zb9neg;^0HRRtKH@P!ewXN+3V=o=C|wt+-Hio5c4G(=m(e;Wh3KrJk$xcC7HQa@OsB zdAx-iaurUriyVsOd6_ndXo!lJo88mrp9?Lz2ae^yqNlq73(Ucf?=L5(Z#QpCqf^%Q z9u?hLwsyhvYGdK<7r;S_AaP_2c=vcTZ%GVD3e@#Cc@OSe)#My>ypeSKKT5vD+H1^4 zS7~ez_0!7CMs$??$iWad=RC~J#JsEj4;014(6QE9)>ho1IA#3x+Z`BR={zz%6!=y@>cB!1D^|-<9;@*esxoN_ zdS09S%db6vE#P3U;{z{!<&3PgPJR_M=gko{UDAqq7N5*s@CE(dtg3LWh`;_ap_7w* z{Jepv`%b*`r%N`V3;@ZnveCkM9wT`+1Zt)E?-%p?#bIOZ5l=6kYW@aLi^{dD$h8o< zI_v4WX?SvnjnX&n-XIleJhL+O>Xa0#IeC@Bihqm@ay9@^u1^aTyZc@n8%(osjI;_Wfs$ zwdGp&Ma}vAy`a9Z0nGjVJ#ne*bvB;lTHD6_8pQ~HxAm1o#&~vxOPcD`-*9uW7Gq0YVuKFW zEgbkaWn5P=hqy7W{l1wdwZyz^RFw}nR2Q(qrv4{G!EG_Ni<*-!gi5FL!*m2)7rM73 zq6Qc5ggp41OwUDQC`1aE~ z2GIKLO1^dT%dJk{leRZAvIL}19&p;!(hU3^`Vc4he{YqzCmDxc6{GF}f|` z|L{a+g$^8yr&5gqVZYQ|$=kP5Y3m{*YR^p*)+wC<(&4<&rO*2P@6}^C2{@Yd{3(|u z_ZLk`1j;Zw(?d>z9I*n;pX7X|K6N*-n!;1`wY`*ArFI$Slk$wxp19`{nMPT5ZztIO zd~2wCnG?vqg%CUtz^(%jVBhNI&9=l3+cx5(GxGzymFGZm{;GyO=kX0TcyRz(GOX2yCBBPZ8rL$DCyCH-YM_8ee{%w2f315odECA%E#S2?z;U)T#lB+KI z5cTO`2>ojhaA=+AAb;%UkkSF4Cyt;%$2I!Oj1jb^a|zK_Z?Mmwb!j=YC8Pc`c|vlW zMGSF>TFbq;J_pM}s1M7o^ZNY~*UsQ8`nX7sf&J|}jFtU)G~+r>b!AgkV7BP9Rowq6 z_c}NC_fFn^d?Fkv1(#fzEyMWbxpMOv8b4)^i>bt?pQr0A_F~BIiw8NaU2JX@eMN5% z1LF4~RyPR#X0}Z^HADNVGsUN`I_BIE)g~iMI0f(FOt{LJmMTo4Gc5h;+2v|7=Zdee z$B5oX>K&4b8@DL@-TNC|O%;<~;x$QOKOVvs7{He_pEkkGl;dnwvR%c%t=2{`gzpmOZgT z9Hc%(8a+_RXUS&}lNl45on7X6%HW_bR2m0hpyESIgvh>k_!{HUT&Ok~0g^+g!FbsF zLVF44<-(a7aml?Sr%vlp;R{%?Y2=7bcBrVc`_E{m)cTI_^iOwfODrzTBcAHGcIcB? z$#F3!3W@x!P~AT|Su7JX1Q6Ya*?!h*_hzS8FpBnK$X<0xtXu@b;yCJNf*|RffqDL9BVo2upT^)Eb=A8`B%~`j=`k_8aZCr~K1$1a*UM{C6C@Izn z#rrx=LGSM8Z`7AGCts?IiuDW5BDDm&yK5l7MMtuOUXQ+pLMa>w%F+>SFw>5rTC4CjuPYkS8Ot`wp&x=TW26)QR5bn z7coC3Wq-kg1nK6Re!5wf7*CFjc-U`yW$OwLnYX+p;=GZ|OW)4sS|@Z&nQRW9WY72{ zSp$4N&J{eGOafwS#%Kdw4$@AJ-77;cNrSGWR+YTZoFA#{5Byuh{r1nq`u{5l^1n$S z%HhAJ6xQo9P%bpb_SW)*{7T0;Ivn;pMgAsDUX^yF%TLO|L)u}BbkoVtP(Q1edc=bs zYyFeszoBsS%$8yo^pO1~F=A<&qy}1cT8H0fjz2tj-FtAPdu54^-5YOP;%;X+2^|LH zPl`Pu#Y&$rUns*$A9$@N<91K+J8Ux7LV>Dp6cqU7o}v;P1i34rWu(z2{=mqv1#`*U z`K@e-48p1&By!VLX0-=XL`KhGgEGd-H7{WIcJ(}X-7j!qaQq-+eE%S>{Bkjw1c?br z+(WVrfCF<}*c|@DWqV})lLgOXrUD3|p7DOvu|7qQxR&xkKX$vUdEd0~Rby#(UhS|B zqQvxKUN`O|$gTb-4I1JcJ>-Q=0ejnJpeL}$fbd+t#1?ntonyh(yzuzgh&y)R$0{|a zQGd)KvHIs#G8eYNXBAX2rqzTR1&RGuKnd78BD9Pdo+P* zfJ^-uFu)1KFR6ZOMq>rxiPng(CWE5`Gvs-r^8}5$P*QOZPviWvPt40GJXvUL26X^L_xs#FK;zr~U~Zf@>E50j*zXMg zXqXHnqPSqI(PV`ao*Rtl-HE);jEgLd(<^Lop7Fp@%&8Ck?yv*n1X*sv^-EW@_Qal` z-4tkbqfSrBUXS!9Sc;(pM^6ncef!W-ZH_W+dPOHR_8d&b2@h`gF^hbp-32d-M6*Y*N#|6gB6OQ81|lJ3g1*j*_# z(8W%r*hQQv*dh>F&3-^PuCE=1!Vnrr1a?`NZhi$K-G@giCvUxuLj>65Y&ivWnjXDe z6`(>!2s$0ipr|lW;<$C0Z*wS;y8G0IykE6)Yp%12-_WPFK;u|yV7TO_6lvH6 zAOYjVyZpqnmTvNU-~NgNWan1%$V9-w$Ahwd8ydHde9TlbWAmfWPZM6+KPp*?i29m4 zML$D1j?OaAuS2tb!9l>Z~lu4Rkdp*3n9`@u9kaPsa4_cNCUF^y)!W?atov(Iz#9VE*=!wC0(<%za$INwx3* z*p>T=Pf$PE#k8bFI?N|F*1E5%?p6fzvu}yF9o`5Hg#8pH+04(g`=jds`v8Y~gtQn= zr1*u1@DIWylc)@-(OaKKL3LS3l7}8)Q^^i(jP~!kCTNl)DW0|9cX>G=Zn_=nVVpI= z#bg8-+L5^x1{DkV#6p2^2dvt}0FE;$xpWk}m~^p*%>(wSoJ z)#T-7)k~^zf>-GMe_mGexYeFKaWhe0?(8n`LXq!+;UxUmi$9nTT2G@Tn=@q}HQ^J37PiI+PHe46P9nOTXR7M=fGU9jixT?_qClq`dXN zcY3m6shgXA-v8N%803M^V|cOAT^1uqhNSirYdn^X2^j?lH%hBo>-NXudS@?TKKWsr z^GHD<2A*?!z9JZY# z1H)f$xaTEeZiL-#Ks{nrTwZ+#gbAl2vK<)WN=dI_`RGI-#td=x>=}}-zs`mljf__j;eiqj94f>VU1)V%OFt%BL7xxE(D@|dYTBy< zSx87FTe+}lH@4GjiadVG{I0rRC2)A}8yf-k@7(WWUUH}! z6RM;~jK=1NlVhy%m~P!-sgT1MY3-NjVWz$Xrz{xUz72HdK4y1~4j@0554eARV?1I6MGv)80n!H#u$RbcjJn+9PN=&}S)_f< z!JHq6;-PI8?w+nNs4Us`Pow!}=li0xK7FSi6qbqN+RF)SXAmUf zg&oqrB_$!!P`}N5D5agE;d}(B-J?;>9yf?_AK9*aQauASt(HfB%iM>IwIGM~lYran zy3@5*W`)4dGV|zp(H2LTsOqzTXUrIsn9Dd_z0>WVIW~urBLSm!bp9LNc@d#Z)1Q7A zoio<{ojKsHecNI)F7ezPv(egkEnj69ii|bG8;G2}lXq++@3T%E{bgX*Kj^Sl@+gkm z&~W@7+TRISGoQ5dFQ@v#&~CtG*isuXUQ(-?a%5!czJ&;^OeF?LiFB zQBbjQcG-+o*xmK3HL{eYol@i=+V=b-4BRhx@wDTVin?~m;19Pia-_~3^y%X%QNWf@ z&F~&l?X08P@-b{;E?L@#3kLje?46yBlO_)TvL7tX_K_ccxfMtvG4cKhS!eE4IZ zk?;9$V;&ZqWtRBs*Q?je7gI>P4cAB;4B}OPROHfxH>l?u+hv@lM?Sl26rjf(G9v_0 znR4@w8gUYNXzf_DA*^7ZJ)`>cY}i3WF>we>ZZtfuwV+?iCVTxOM1ri44@%}Qq9vLV z-WZV+?2)XvTRm=(e_mWG=8@!`UYWZ_N@-`78d_xq+3p|%>q?)!;EC&COTNp-WaLH@ zg8l1^a}|e)?-!;+%U$QIYp#0e@T&Zu)t3>BgqX_wCsLSv&fkA)Byl#Oo9fiU6Cz|= zV~K>(b^6vy?fQ!BcYOwV!q-}-U=QoJZ8Uf_PXC%6u4;Mh z>{*q1)#hY0AXW&1+FqqiWv8S;uC1{}YyU z{RY2G)2xVT1Tir2*oTRgw%H40-25!TXF;$}n38^7xbYZh&UVFQ)5I+a&+a*JLZVCz zp(3B8XCRM)?a^Mp?)yZ$mcTf))1-cV$e~c|s(BfgEe1I1GH)&Duq z>6i=ppOQWH8rklv;u1eq(S1e-YSExFMF4tL%f>5`N6x8x0L|N7wN0YJa=y?jsQ1vR z-_j~4>>l(-9GB!Wba9%cruIh_b(PCtlENT-B&7ZH2S&tb-kPJv!3#ULCND%1Uk*5yn`iCeQ*?c_v6spmw3RZ?0b>kOs%*w2~zAvkRI!M{F@GVGQyTyKx zG(|;%x@^L(fvK6)nyPN)MvdiO!fV`Sr;srE@FyGr)hbS0+D3ULz+J;97W!{hA-jRg z^BDfM%_&C(a{)5Az&49HLjQ2=^{BdD zb&0JB^8pDOG2CPQpWZ|B()AXxHx=sN&8Jk{%*hsC#pLhGe9#U93P)W~yryj3IpR8K zPm_|G1F-oCw-1>9r(aOAyVA!^@ta zrycDRq`MKxqPhRexl9^`BWEZ&3Av?=L|cWQ=&J0VS-%q330Sgue!G(I0GWXfhueT{{b^@}I{YyY)y7!*s@z-;v*;YEAzehu>QuVK$0Kx{uaHB1gZ@tVvnP7U=*d zH33DSu>{&*maY2eX+Q>65F~nFLX6egqSHzl&8AtQf%x2Hj8wMs{x*sLH;DI&PV%h@ z#7(gDM_}`6>S~achuD2nFu1B8CAO}n7QQQ^?6{Ac(A7hgg**Aj0uRBMumWGDm4}14 zyssA*VAJTd2`bn_QQ%+#b@*r(mPO&xuza(FsQ=>Y9_beOK41b+ zjlry2ZEfk-#Sz)71o@9Xl^8^9TYmQ|Fb&38h<9sYLVQNGkBqx|a&Z;HmBe^`^-6w# zw~W2NfgJ6*>f$D@Oxun>yH15C+vs%okcwA0z`V*Km}5qd#d64>2i;4wIu{_?cb$|X z*GatJDQg@;0rFfjpzwUf zgiw3h;yv?DNZ;BExzOO-9N46F$%j$G*PHYO7&~IS1LKj1D(KBfBrc%d`w;royY4;X zK_RFF{L@PQ9Yj$hDJr`C7Dcy79-Mz2Bd!FSI}Be!sw13KrK^KN=q}cIw_O~TN9+6w(PbhnBpB(}2|jvarmo#%Y~vtBL3GYj@M)m=?UX06yTABSI+w!%%(GBT1>DjM z1}ee)6xJJFnFka#Jwe{kjz5m z=uwL{g3K4+3Wl?5%Sdu#?Q%-8kqPnge{IX0)!M4*=3_O=ronGKDDsLGr6|{;k6(?q z_<`w4SRue_(c7Lm#W&eqxXMW3p=236?4w8>{cS9!!cY_SD#( ztIwW_N~{8qWftj)&RaNI0IX~qoqnzut5NKPlJ5Pzx9}<+W5YT@Ht}*UIb4*&!#X!HRou2yx3nL-;}VW7Ffg|(@ladC zgV~l%Cn#2BGV;^{S$h2_%P`9O+%9+p5}ZQCTKucWG0akDRmyo3cu%JigxPbLaMR(yx6ELH?|&a`oP%CSSi~IsaD}%7Ix!pHDH+ zVY?9~$Pj-t^GihrL;QHr%6QlRS&=fm2AWnD!@_Tc@13Y;bBsC~sS5*jxp+Na*2V zs-JHA&z9ez{%nWgGN=0%+ zIaV#YHbb(NeOXo#w({WAlO04}qN&8#8E9fGRE(=@|0nD(s;hMygf17;ASgKCV_7nz zc|CAXQe=1`91FyQI#d`;aOt5JEa6(Zaq>c)n2LB3Vm{GV{E+?|{Hg&y@~CcP=;|3s z;9C)uwtZ2#XC4ze=XzM|b@uMAq`>fm(|t65$yTq2pKBga__9xjn952D-P+AL|2|?S z0H+;0W(s7iW|m?z?Mi)HE{z8))$n^zb`=#Qvz8(Tv?3Hc`U>WDz#Z4Wu2h1j0G*%A z$uGNG)Dooc zZhhP1i4w2n7Rn%+asp}@(7#PbSzhSR8*4xX1a}w9HqeHNlqP$wJbLHwUv#n+@=aOD~ZTvZgkeTdGWF{N(baD|1YRyAVwh7gcSF-=YquLpy{MnW( zFH3s0e%NfRzr{Gu!M5=Ijo@+ zg4t6_nU!99#esvx$c=W?Z^JeIYRAj^6`oAxOHx1X-gWpgydbDOOC{^}EeYPyYG>O8 z=3FNTSFy;&)VA9hk&1{JCgV>rT85KPy;hc--G6PqROu<_4W{NDxT{iUu?WfSe71rO z{KXK$L-xw^mkAzBJDd~eFh9TZxXqE3{l#b5skAb64A&Ml;-wrkEe}jay>Uqs3_UJ$ zXH)q(8@+c&vPFRWm&^`N>(_72ggNf|%38)RLw>)arQdB`yp>W<2mx|jbK@+{4KE4r znlXT=nc($6ro~^gx{OG?J`PYd^(|tyCpn^`2iO$DY+VoD+Pft~WP{RL9LRmjIv%&D zM+?47Z06tj=`jE%#;kaHk7UZuj!wl~UKg}#c{EwWgt-ddwO+jL0)l3of+4++@%x$# zz0bY5Pl^7ng~?z5`M>|({{tA4vmyyms*?~bCZ^!|F4bahos#g(ynyLL!8IqvOJB5r z#2j{gR2BQ8!kOV`ALGXyS{3u{S8@j5#^-$t1t1va0mT-E8pTDwtjfqqEr@5S^4S=% zA(L*VQVUJULB0*$h8I)2xmR8qzaN^6)Fr*mqEQv5G^D-krtQK>a&1ca`QQjLuyg9@ zwU13#A;_Iv^7TZ@1ejxON7wVL z-g{Lrc8Jj{(u-XBHQi$>=@tbrFzxTx=Nl%G>AcpT2~ebFvUrq5w_Sq?$L_r8(ari4 zc@1Ip^;_aX&*i^zx!6SY&vluKpY;umeN>%pyZz)Gm})j>6HNEEZYCV0!~e&3rGlVv z^SSLn1c^&?i#mKAvVXfV!tD!RNC_y8eIkX3)cgWO7|zNenK)wm`eu(t%)t~G3qCxz zT<&17t?>AZ$Pr|IZ)RnEWFx&XW^FT%V&{obw0+7V=*E*VCI}xdSl+hc z-GyeNrjXxW!Y>sA`#+hTiS&NO+PvElmb=r~r|PS@8@0%=y-VsECEp^(`dRX}U=-Je z+*CL~zQrI8l4sUn_PnxqV{g`lBHwiXMNQ}`nVXPZMLU;#_;s2zenwjXmh2xrv=GbF zE%UzD+zhXxokH+5LXm49G>rN^BoI>5TgSBq;Ir=ri0YD+>RoBbqpv%BpOt< zs&;I)@4oZCtOFV!lhK~$Hzd;k9NA}klR5h-vUJ(@R@SOR||K>UCyzx`3dh+VPL0Fy?-dCb=j zrnE^#X3wXtEF2#qBEjqI9O{O!1S*#K$zhZL!_tc+5*NwB0=S<$7XuFK1gr5bmaEwP_{ ztCsFOabYn!O(uE!j^)L~vqV1WaznqGUTp+YeB!?OqhPZJWgN}F7{7gpd603`#~N)_ zhkAR}6WSw_w;+g6eUT7#81Imz7W3V&`;E_X?6);IsZlZ}Q?~0@tyKl;iovzPWV1^f zxj$<;)=^>_m-iMzM$dhlC;oJu{`xn}I7D$(Hl0tArYZAvg@<9Bv17^T(ifW4(!B#M z3hB=Kb50e>=jsIa;jxY|3Hrq^qKDdv#otDBBqeZ3+x zl#-RrQj!Q%K{}B&bCoA`W-xrDovkbWsP1*|aDTWxC*fYZ#}xlu0h`8Qw(5DA+7VM) z58-BZ9Tm%UkL6Gw&0_kuDI^$LZ+bPHSo!@Y?{h`*un)ZU9jW<=q0X5WRPx2icP*m5 z1m^ka$lF#Jr8wkJQxwYV_tD}Tp{OydL>6hF%`w+3s0Z zb*tWPn1v5{d+EmRFHMJEbMDqtL9(nh%Oh3Ssrq{c43;V5lq~N$Y4QBmYj#fh6?aOY zil$GO3^@%yFSy&H#{*A!Ki&}LYy}5fINOvIqlAV-&mwwKXcoSX>4hAx#pgd?b8Xbf zz~k6URrZ5S!=;b+wmDo!nlC-%ioQEZY*<7`IWy77RTI zSB0NX3TXV^FtPAZPFS7A+fy#@)1*nFS@@pX9L=}ldu~pz#TXR`+ZBE9&)r&kL!nG3 z)YiPZ{CnSJ0hp*ki2)KfDkF7GcJ09QyT_eO>3j&yFRNAm_;V0$|e6Dsw#5$?j^e;l7WrBD1N_=wP(LJ&5x;b^zCwLb-J+i(@2*X5Q~O+ z?Qq>GvV_m4KaGuwXA5vY|=~qEj=&J2Rx#mex>J}S5 z;o!me@88}a5Q$iMC_S4DgNzdMlMq%386a|lQEw<9!C9;^V0>Wd51)pa0U!A9nZMyt>)2X7RXF+G> zi;=f04e&RHIkRi*5+3lxaklU|H7VhL_YtkfiT2x%F7EF()PT!z$H87`ji~Tuf63$Py}>St zmiI%d6B*a|TBOh1la{G%%~7sUuDC7C<2)8IxDyw2AumQ_Fe(;&r!97(1qBVQ2Ui&wL4lBrcb4*mX-W$}iuGnYng;dm#R{r)p2PV}4`F7aTIc?5Zy3UZRX z@OyUk%hLB7>~yiI^OXtA7c1YCJ$Ak=#``@m6qAu3o-Ct8VZ{9zhE=E$;ri;nG(cQ9 z)-C*5fLS+`qo$7oBaW%LH;0u$9DnOuCvvqHW)TTvRkJpLdKdZpTHG1WgrC<`WNq5p z7;+kJnpWsuDc-%@j(gBADu-c7E=x>ky6n0)KvL-SVWtqFJA08>-2iGWwdJj$`y`wY zO9fp882`<-v(YpixwCWQxpQn+4dj8Vp|QayX9mS-!QQ|Tv0Ivr+!M+^FNzVoEw4{8 z=a^f{xI%Qs&~F>#?qews2X@F+3L-STBP~ z8aNqxGUwD-CJ&bEe_|&DuY(iCLJj>F21|xJr@w8E@O}C`as$dyY5#-D2;vE`htPIg zf5`FTyRjPB#Cg_;{f_YmuD!&hA&P2r4@bBf%<6XuPoVZ3T(_9s6^Z50-Awa1<86M| zeZ4io^SmY@o)(uax%s9M@RuQva*DFOP@v{rl}(l9D7@LWoMT zWzU`#%34iAwlbFN`=ButO0t%H8)b?_WE(qK#}YyeW0?@LjJ+8QGv^xL=bY#8e4q0= z=k#&M1PF{G{9V30LaNPn zG=JE>Fb^$p@2q0lMP(VW+x?>#wXL_G3g^$rR_e(m_I%=njPOXq-MMrfeg2{Qx;YaE zVXL6EeWR6*11D#)B6Yk`#dZ60l9!Zw*i1vW1F5n38)eI8)l4rY+n(pgAVz^{KesO$3u)dp(%qv2=D$?%exNJc82 zoyEO)PnZ}<=*U;Q5DN=7u6ohT{fwoK)v0VWki_`wj|U&yxq8w4i-KX1LYG9$aAULFm#vDXcWSwOI$*Vx{OgCjDii5n+!gL!m65W>)qB(FXz226q!+KQH`@* z2>5xX=x3|#8{$9qB#Yx#7L@lt!o$rHqDfBos}-kWmfs$(0MiZS@Al#H!%wDoYEa=2hNipx~ z&?mmz0Di7+R!A-PHVIjKNh=2w1(S z7!xRZ^}^M~<^zV`ava)Epehn09lMf*TaGnibQcshpbG> zRDWdb>dtNVAFg5`f2C*pLEi}1C$wSCB*-bipEshUu>X~VcC{O~u%&J06O$wavZP#m$os6ofeoS%y zbY~&t(j2cQ?N0j@wx@`sOro0vbbL;yjVh&KMmr^QV1^&&F~yU=SJ6frw7vy(At|o#mH`y z^RF`5{~FRw4#xX5Ef!y;-rmGNN$dRHBHKHdKp|P!N-L^va2MS7o1Y3m?OcqYkX9<{e(XTZr99h)lJ;t)%g_x-)%51&X#3#&>N?cTQVkAHVaBU&ezX*+41pKIwLKN z1;_yuXXgxrm1E(E7Kar}vYXQ|E4B_FxI+1RYM=dwpZBTx zS)+p}{qwKo0Npx*rr_*S;+UR8h#@c#RGYX15pIx=%d38zHDi?NH;mGNWWy- zD9C|CN2q6Mk(%%I_`mz+l=MG*f8;2O$Hy;Cq8lq!zFtmAbluiN5?U!tg$aU>IIsok z-rcsg@3$lV*mlg#wb9}5!$wHYC{K0b-D|b;)a~i%Z=r?6_jVoqzXxO?*zclCIH$jv zW6bW-ros}%TP)9>toyTkuH0t3#{Se91Q=;WpgY=?s>^X*a+_C5>1>??=lL& z4^ULIZlPHovp9~x%W+R$Ojb$v#bA?;vHRBR9~?6V4{`^Pn}$taM(Nb()9%QOjjOv6 zhf4^J>vfGPT6dby#KpZTVE&8kFX=_UEYYRlnWo-@#-fd$C0^eh2Lf)O36{43p|pl~ zAnkj9oJzR#iY>YTU#+$fk&Q9XJrFwmFbv2efb_4xi+pNeb>xFmirgwuT|1nk+b?sE zs&dV2C|db|L)H>*J4`+`4)K&fl%@ZJi3{CePb)te$N-_x`wLp;2Aq&kH`@rJ+cSS{gRxi zU-mA))9FH()wkXjNKPcrZO(&SW=sam@s)d zBl|N}6z}?Vt?2~>Co`{b*HMqnLT^xwb%z~fR0i$tG2|b^Z1RUzQaYwkh8+bxa=r4G zWmUWZjA~UK636g}I!e+YtOM!ZfWu5At(tmjL@Mie0PHCk`!Q1u5T&mFqk@_F*;@cm zv-8OgW=SNyd6i>Bde@to!LK9fo?0dX5isU*S)Y#WZir8c&LX%isEEq+95_= zZQGTXb4vpyjpa|BNQj@5AtGBMDywtE^Sg@2J9E_o_spEM?V_T>lX3&KRy#)hVw6}s zC7TA(gZw7Oy#k~zk!&r6ojsXS+udwPGc{Sqk~A%p9s$+uY<|F&R=nLzvY#BQ`X3;N z3X%E#m-Z=wW7Z8eDTd+!bZwu*B>YP5Uj7B&8V@}} zG$|Bl_uF`)+ghoTNcL;FGu(J6$qMKJx!#hm#WIo)wF6UsG3ECcd=MlKA|6%!fm)P8tQS= zXTdel=mH24Q7{n@?lJo9W#nC<`o*XoDYATg}RCTPBM{zc>R<*5&SA69VmgR@gc z8K+T2;8oQxq;o!eZlD3(f$zgvQ#k%G*K)G!>u@bg>Ns<$KDS2i!NGHUi+YX{CZ%}M z^$}a)X*~`rH?jGKA6Au%Y8K`(KF(epQ%6sAq#djGUfTp8SIZj}M0LQpA=#+9!N?XI ztctl9n(Voz_>bEwp;91uI4|*7mA^%a@_{fG><1I^o0Uw3zdSbrQFxnwCMSh#aKusRg0Z?f+AmfQZ>FBw8Ev{Kg#&x z1?JcGPld7H3je&`p`zJyF9p6^axF>D0jNa^J<=r zj)UliAM)oH(_7Py6$%}ZXj3g7L~yc2s$vuk!H5aJII$@xpLt}$!;)K_a|mW%ui*nG#|@Uh!|eaVCBFNhPq1#+n4X?viB>jQR87k5m*3v+($X#)v5!Rq7^r}lKjHTULtU=FH_w3 z4=3hfauM0wRvJi3Rc>AS^0vv%{k*Fo1_8IK4gn=D%+`G_xx$)`gYkYjmNq>yHui9O z#Dl7_++cm9CI4qS(&iPi3N4hWhWbXyF2udqL@mc48ru=V84~oilQ?&zZD$k34np-U zb)p5*Aj_{VRAWZEEl%qRDaqoK)*+^yCC4jKSB60+xJ>F4H-VGAAEmTQT}uC=Z~ zz2EoI8!MnikZat<-i+a!me-5b;y1UyvDN=h_fFFpWDaV8N{y$8^FvD}H>Vbg3I9|g z)U*>!h0fg9gGA3YTx{TcR04UCGl#nyd4{$@mFc^efEOM)fU+5TWeFLQO6;L;N~oCJ zF!EO0Hb7RItrO4tW|saob<$Fz!J`rGJ)?{^>;DC7?Ehg1t64-uQu|~8#TkcU??xUr zb}+29b0o0Hla&gAH>RuU{q1Pim{Mqy=JQIbv#gA=jUqhD;&2KLj5woqkIz*DN1tID zZFq){`Ap*ntcAAhE|&txF-f4vQ{BOUoWd1}7Vn7#>@RX9ZHtpdc8{j?#U-U8Z)kK0Wx)c! z%w^@xCoXl0^rl_odn;K%J3mq|}P~qBvfRb$OJkpj@fE;yY_=qo4 zgp-OeHu3O66sc^ItPC-NHHDAn%b}Ps-C!e=61Kr zcX{`9us+z^s<<*8Y*be+;n{6ni z5_*z*dQ43(7MU;K_lGbArrj+FrkmqPSAU`lajG3&il^26NZRhQ$7j6`-S_PH=J=<( z8|2>(LRO!ap6A3+r?-KndW-xnPt3Q%2KXnJwzk?%*pW9X%U4K%x&1;u1{z)h0(cm* zjH43KSXu{SC<1!+RNeDaQkjGL;q=DA#U!ksL~ktW-O;hR1c|)Y+ww{!>*ezD zZRKT$>ZfAD`jZpl{o)==Br-&niH}wy0*rk*P7rKG=B)9xVy4276t`Takq~`k%-opi zJjUV5Pfxt2;^lt5_U4b`=6p$I;yB0XZ2)*u(v>&fRhchUcAc0OSReOE4e63QbyNSW zSpcCQx>D}S#1n>P7=4^Vs7zwY3!NLQpsu%*ekrc2Iw=t+5EH7z#P~>6JGl3xxWcHv zxL-&Bby549gn#7&-10XW@ECW0$`&RxAn3J%vbHYhZF_zDbjRo$53he|tO4{h_$D2s z_>s&>vZUf>BYeZg;H=J7uz7f<*OUs!Jp_dN2h~vf2bU8C0i~1n?_|LU0 zcFYTn$yI?bWALx4Rfc|=K2Aa4zZFD$4|57X4z0}oEI(82SD^sGp-^as!hurv-ZSNV zL~uTO2)R(1#7JlC7vQs4CKNce!gS~a!N4$!a(Y?SEnQcs$eLT79j_E9Obmd(y|m27 z^o3cl#K-JT-&stlH`7hdPLU+?OG!|>&-cjth(dMPOni2?q63gG?cjD^*}nhNHs?iQ zJ}?%BsVWJQSuPVc7{0kUi|Q+Kkm_pi2t{AN{-oBoPlY=MD1WnZ*kNq=yDSqk1vBspHU+^+gmHp{ z$+`DqIELJX@`pjWr<;n?Cp>*490iviJz1E>BQND6mLkIYGe+pmzNoXSf3NmmOJ48} zf>$mM;?fPyWB6OdH+;bRyR7GkLJ^!$+1YD@lT*w&6oEx%^&j+RbjDkkeYqB1F)hR~H^QjBT|@7O$@J;^|*;A>~uS{66Usvzsk@c$@y%gN=RPPivwck!1boVAKVYNw?|0UXHqR+N#)I$ce!)G>sL_& zXh@2@v)Y*>ijO77>8 zkZz;!@d}Ja-@gmRcX3lk&M>2bx)CtN`SNziJKnlB)iApt(W~jFUti;Q&u4OuP7@K8 z3ZAP#RU5?$a~-t}x?>&erPCX+Kz%&%?EOrOhqY$#$dXG3yA{hx6uxgZKT?m|3I#!# z&D5lnCj!;F+{P5EmxLxIIYi$eGjHetA?-eLCD2K=1hGL^0?~Tx?F#0oek1B*Sw*`Lw^zuA7T>h0@FJln=A+M(O zdFEH~*kJyMZmhPflT?r0P{FKt=00dFXB}(0^@zOjEo|q-kFlh1wh$#a{iNc;c2>}| z7EAvU-#%9+<%UiOQ$3SimC=NwQwSuKQ4FB z8HXRAkVTT4ioOnn1TcF4N(6Xpfp^dTv9j;K%GoIW(V6>UY+v3{M})uYigEJ=I#W@) z=P>X@XcF|3MEw^Pjyg8M{KpPJTu(jFjy7nsP(pCNlYe5t6P7+#fNdU!XqTGbRHV-l zfd&uodPoUTxHGJg6z|`-|ILOtwoad^Pd6Uo1&ox?FxU zTI9)g`PaoR+_WodTEJ>>+ET3YlLWuxjiKW$%g79@#=a(h_|f_bIMs70H6MZQtC8pK zcf5ke2=GIP{0)8X_vaNKC066u-aXnz=Nbn&RxH}cl{w#y-kyEjcGZ|zJtm#O)SKQ- z#EW}Pzwg48tgimX|s{}lmZ|x8JBkV6{_D;mOESXdVZdp)5^AiUxo)* z)>@Y2cioFzwOokvACTZ9XTs7~1ci5f;)#!XHemj;UR~ZV09Q{mQ7_)ZqV5)jb+y#$M^a!YR1=dgRFYtrDo;bnML9x~rnGfsMDG=EEO@hJ)i(Q7 zSLc|I&{93I*aORMcGvbb5fS>QrI>4@g#-9N9Hx$)&;Z4Dtcqy=N=&a!5eeV}0lx5X z-GhCs)&zH6oGYSA_3n>1YrOu;jr(`N;P6(j@ZwGu?Sj?bJ1TGn07urLv~84B+ba;s z6UeoEUsB|9eZQE7b9W6eE{Y9*5L;pJ#V>SyRj>30HTauB%tzl*hv>x|p8SIN?mg-2 zzj-2i$P8%)C^#qdN`Wg45=OV2P91r7~n zyFXuM#cS+oY7SMD$~*zJl569Qs=ZvBsSxjOv9s&&XS*vGe=JcieA<{rSwGIZMU~r1 zP0c9g^1|TcjVYQ)vYDnk{cS?q9g=5lpigJyecfA;%zRqX1WmZ~C!#~xiN>j~pV@0R z(Xv5Zxy_ior10_khmn9Nf9Rkeq$!S_eZ`q4)c<>0o((CZ%-l=(V*32XmW@(ht1?~ zjp0*+lS(hhyOX#X>)0OaJd#X&P&_?Ia^S`|6wbI|2>p9znchA_3EVa84XXS z;Nt84j#LOFr|rHF|BWqVrt>j@Hy)39PI-D`p^ULX%v=)0&%zs|a2Uq5xv81?tz_1VDwkb?HuJ+P3fuksznll_Sp%Hp#EIGgxB6Vl z%FC*nq)X&3eF=^nir-nah0G3| zAp&Ob#@#8Wyc)}M+1y4hAi)7BJ&83&?zLBLcBnzbmT)R~ULZ|25zDnBLJ6It>Mj5>(bAGIQ?%%gnua4$2HwrZLo>&&sI``K9ud=VHP zWN8OYpVnLKqG#Q z8_{_=+MX`%}KwV-?0v?CzoqlB5!p_*zeD&bH)AEa=TX?pg z+fc6Jk%a|4~9x4jSt?_|N|N z6H2Ihb5TKDPDU2tM-vz*fd4XA`BT!=w7FFe5w}NbYXr$|&i6;3$C7EiX5`qOQ5XYw zrUOI#0G5?IhUIoRXND6x8v@i*I4dUVB__uuL; zgP`6%k23@2$1faWhni#0Ngq$=iEA9!MkCPKO@JIHth(yb-3V?SSSky)rn?<%=_l@I zPUl7kA%WaYuIz|(VVO{8!=Gh@CEut+LqxY#_>l?3&>dsH^N!HiLGy_a`oa28{*a~n zclu%nLhfkKH(U@!v+Sc%@Ck{?XJyLzWZ=*wPjVjv8>bW*^#D-dZgVd$``98iIRQcn z%qFWUVFvlS#2G{v@ol;xC&+~QJL{(TTXscN&9%}RnajCf-~qIrKBT^9%uM#yIl3VfvT^NiDDZgq>Q+K#C0 z59cgef9Lg$%0@q4{2iGD=(^N4XK%dZelDi^_(gthg#J^9M{0p{+|(jn6Y~1)=8ub| z^2kR`u@>;7AdVqhIsE{56sFu=b;MClwvtcD^uMYC8O=fwK6>or;q;!3GwO1smhN1u zIyth`4cE`6+#(XtS=qp1_&4TfuK`(CKIA|mbX|-=FuWbmxZU)r8GV}Z>J3dX+b0@m zpo0dAMB*>GMJz_fw_NxQ7fWYa^5f3q)!_5la zH;|&FQ-c?(G*d&4+mjyM<9ar%!hXhU;#u~zrWx>rl*Hhf`@-+o@Dy2M&TTkw6chqw z$EF>9T0r&9R$3=rnBbXjP9sXPx|;#=Jkd*b96A+o{9)4;ZU`OiW;u4#dLn>E=)Aua z7Qa2;e80``uB>(hJie5*mwqt7*ZD@9EceipUjx5a>uA4eZw`hm1Bq^WEEITpzffk| zc)5OV6&ca_t0=*zh7DD8`Ircc1eKT(E~;UuOuvN`xoQn!n9A22hUDjK1q9zPxEh(Z ziLZM~tFwQPGkSHvj^5`;nxk#}fB>n(ga+B+4S;!-Y<9?qWO~h)fRrA&n)#TI^P?mn{4p{_8$E5!+ZTSEn<=^U zgq3y|R@>;C$mcf)k*=v=hn)+)oXq@V*?pl3W++D=d`E}c6l+W8I^Up$DdVM#R}URb znL1(h7pnYm!%2tSpMKey9gf3D|9dn~yG3ZvJUIM}QT<}YL&(YdQ#UvpB;&U`?&+l5 zSO;YGh_rU*)sJ4~Zdcgv7GeeuMlPjZtbace_4Q@#deUjQ_|%l+e?`={7$(P%$l$^7^yw8lLs9zTa|Z|WBQxm+Jq5Furd9}{4ike9Qpt6; zG3x9hJVWRU!FD7?qr_R@%?c8H&U28D8wHT@7Uw&!9ft*p`uEC#iD_Hf)${;%1G>|l z_E>h)2XumNylpr+{@Y*lLGTS^kJy?F2SYLJ#<zD*#3MJK9vehZsy(;^Ni+j*w^A z?G_$tCF1@Z$gIkgx3kdp9=1w%k_Xh3l?U#72$-;uIs?PD}-7zUGILjcxVxc@{_EAaf#| zI8eLs+rYzWeA|mz$9!n^-#w?c@|{ho*~6wKOr&P59=Q5>&JKdGl0+gnDUmc_4w9Bv zXK6uX3Xb0%XxO09x|B+?of?uGrmV+_4*G*Am@gl3@zaPw&*(6l{_u_;L-STxvDZHO ygZ#7U^hVsd72v)k>~&jq!Y03Ae_!$K01bmXsPY@5>-)gJTQ~Hs=Usgm_&)&nP_mE! literal 0 HcmV?d00001 diff --git a/opencloud/charts/loki/scenarios/images/img.png b/opencloud/charts/loki/scenarios/images/img.png new file mode 100644 index 0000000000000000000000000000000000000000..81ba701da26a046040e0106e3be5c0b35a1d1f8f GIT binary patch literal 39609 zcmd?Rg;!hK7XN#y&{EvJcxiDcQmie(4%R@6OCb#ucZX2HwG@g&N{ST^?i34!AjO>& zm*5sG@HXe%d(XYUzu^7e8v_O-ve%w#&b8*6>-$-ouos%j#CI9){_)2j#HuPF?LYpw zQTfLo*SrXC1MftX&U*au$GbmNLC;=#nQk=RDP`{PuD*QLVQcLCP@ulsgG+$NQGRyV1Y`HE^Pq}W-t>Rubx^E}1tzG?i2jD$$=Oxb%s6IX# z6H_N-3nY6^$OiJgNHn{zL>p%K_v^sCXpnsj8~D**p9YeR#O7@ZbJ_%5eK(K{@r0sM zHW*a&_p9eGl@V2)pYo{x`s{h=y2|ho9Yy8euL!}+VUsKh2;G0(D4zp71yLEk`>)%& zbK3ac`~SyHmDGouPL@+UuqDlM`6%H7z=!{KQTn9fW-u{h_^Sdv+D92a`8<)3xg;Oz zfBd3pkMKYz@OP9K=u!{mXbHkfryM#u;aYZ0Jk=#F+}-_Z^WOHHO%&u{^a6zgcu;&|mGM z&EFD<1=q3X0^dvy4RbCqmGQ9M_8F@W*MjZEbyPRwH|KNZ@z%nYJxTMQ6i9fFCamD| zMwnCG-7qabg!)1?c2rU~T8f|o@k`@}9_Sy78S^uRe=)FY>VnZcu5`IihE+3X_cL%* z0hhP{vovQEeaDX<@A{j9py&kZTsX`JIix8~Z1CHqkjXW=A20OPXu;|~9Wu~VfP8%n zN-IPpIJU5pH9Bp)+4kS|jY#*Lt@WA@I|I%; z=STF+4kzxV78zC31mHXa#>$%GmnbS#P(*KWE7$%M99 zFqpy<^K*VlXu!>QViNEub%gG}y&iX1raGKMPYtdgGxeoTmGI2?JV`X}V9r{<-QJ}HOe?aHP3Mu>kaU505piIC)Y zXd{1+NlI-yR(zOB?YmWLReP}yzw{HfpR6duI=xasi^I>RSGr|jN9FJn&W_)|U+=b% z!b+FCIBlRD_F5&d%tD@z{e`@v4aeF@^MxD>2&c&_h|%7C!97mLqq(8KM;g{1y~%O# zZF3bC1F}ycr>=y+q)?99djm1B3f>vki#b-B{pH?A-0Dd$HpiPnXBdW@UtUZ)oTwu3<&CBb@q?-`;JrlWsl3D*Al< zh#CHs%jlvqtA7R1B_=TAUBlB8$LVRqYuJiTT6yra`}PNvBW1e$gf9kF2BWS-x_0;_ zN9G9keRL3pNr`W!Fos`_pV4+aN)@jzL_;@xl^JZcJ2Fi0e!K!D$^NeBx=ej(u-coW zjjqIa>kIpSJeSQ?mKiPekoz!EM_fYXIa6KgmzXBChOpRbsPDe*2QPN0Ka8$Nd(uT! zoXc%x-H|RNSUKyDMh>;vTzDt@i}Mqh1XrjDYVq_yfx42Pd@^_`Y`EB{ zIkRS|N;=$?2W~d~#1Tesp5lDyo}E-KR&LqPn`7p0kMx*1oxcbor3qp2c*%Rvbl8vK zO_Xn1?BWKiJ*kp%ov-bfHd=Xn=iyO{xR!i*KGJTN65S2O&0M}dk*2D&HT6O!J+b!@ zUTRefS7tFP)A-dq=|aK_Y2JXlrn+4Ob!>XhzVhh5H}uOs<-vx56bx*`NF}VeZEswZ_@=o8`S_E!0zcrZ3Y(*Ul}r`1cJ>nHF8!(rTe6 zCVmst_TO51ek$Wt+@%`5)_eAD&Edc!y7HG6^!WOt;0j3yhDo7*;##eP7D?d)ha~9| zubtQqsO-MA>9}sf{7 zTW6~jAI#!wk}LY$g|&+1nY8B=^2L@_hUdW10ttwFZ2rAs47CIX<~ih%$`3Y|gP?l>Dm#{P?k}gsX9PC2 zEiP!r03*&$xlB{;I;uVrcYT{=7I#^R5ky~-KG`^)KcO{(cPJnqHr+&>9R=$2y=_1T zc(<2C+#r=L<}!Izzp!;{3FZwyb8kaFg!M~LTxGG)JlTum1^+<`&~jq)`BK6`+hg`- z$A;Ztj7e9G^Mc_Zsp7EDGh6aW1=e_TWu}>*wz$jVeiwuBblf1iWIAK(C!`z0ZPUCzGZQWdJ1NlcPKTk| z6ZS*tEe_$$`;7?@{KWgqAtyH}R=B*!QU(3e>2PhC03RjH_2LpU);#cecCVFejgJUT z$5FD9VP=%$ZL?Iqq;CT6r=Pl6EUz5-eR-<#M=-*}sYA45KKtW9@O$DCyN4YXl1fP- z;_N}2JZ3VLisOMOVJAMARIvWc%8xd*Uyqk{u!*ZxZ*8`9Bds&xqXafot60PROIP@% zwV?;5H3V(1TkIoHN6bGE!W!_Wi&>~_KtqFmk@=74AX>*H%jx!g^da|Kg>Taaon{d^ zDd_=SucG|-Kb(&UH(MJ9%-4i1VdC@r-FD-mXpg9eN*)b)P*m#M);98!kz+3;r34<` z==%sAO>t1lSm4p*y{vY5vN0aV=w0ah{jg@VxdBP%Ivl2PW`Q9zS8IcvpF&`wP3M^P zeImhP!6UifdKzo1*<_R(Kq9K*(C1;jCha^uv9$kL$<#N4;yZLAUJV12H=lO-vA*B59zP)N5jJC=4)(meZ8uBsQA?P92qh8~W_T_>t7hH=WpRA$H0ii!;d@RJ zIcu|lJF4HZO}+tLDle#RluxV@tJ>?k&)>PTQ?&qdo(ax5wA?(;cu3Q<&_L{NA`q9S zA&2S2mEsk$UKZ4Za@yu^v5)Y?qlxKCAid9u)mUfmw(jI!r5YP9qP zD3g%4Kllt|cOas_mmyJo9P{lri!%+wbOQZxN}r*jR^><@Svv z-f0e7+&gQ=;SHSo{Yj~4O?(tNN(g2th`4Q#)}kf6Jgp*N+5Hp)Xv#jf7H>dYf(S<) zx=ARy3(kXi0qOQ`p6&r{s^-FnoDQrj7c(8LE&J-es}k;9p9_Mva*4s4={tWQkC&3d z7y|DgLJ33=i=Vy)=oD8cGPB1L@(Z%Fr}1%I2TW5Td<5bFKeg46!+k)jf%~Llx4{st zN7cgMh|*`j;!uIBmrQb1cJGzmI5*i@3A$FTNKlIImMuVkid(@0tcQ@LGwb%9&xk6d zZqb-WyUSaeF%f-8P81?nzsbKA`t&oZc+NJ#aguY2aq5gN7bv7aY zkiBotq;H>HYb~w#;3Qd6X-g8STRW25@6J>(PB=iY&O+(iwKyDwHCbiD4V>(1D5?td zN#h{H1?9cuI}&~4^1Yazy>EzS^ARe?-^DKL8DYqY`N2V%jF(mE>~h{$t1p77B5-k- z7EiqksxYTarD*?p)l6O5RY?s3ZMFsxrCQ3|zKCX(Z{o5SsNZQNNT8~1OE1$71RSx^3^gw%&0Rf=L|g^XZym0L%p8=+Ey zR*DPwgt0|Aaqbt5EJeWa{Rg4vWwIwj_=)I5f$;>A%&qkV;R^z$& zwYG?lm}a@nLhZN7?-nFl-xa5wFGD|i?VqGN-(!j}>x&54Yp?MO=~iSP^+zK{T`B^nz!3aZ!DOo~t*rm?n0Tng5& zr{pXwK^;yek%59w41Eoh^xnWaLHuh5%Vwa4PdsU^&ramWIy@^y2H-lw^PGX`9)r2d zTfp*pp@rKJA|}2bs(Z1mG%Vngbw(>VmX7VebYp!~f;ZBPPFDQw++CPq;x%!QBgPG* zF8b2-MjGM1NB(waiC)SbbrgIUKk+M}ptv^EFJrH8NuqqMeD>zUfUY>D*RcIrwG)ae z;%!z$shBQcPqX!Ej%Q-yg1TKbWu@3X(TyO?0my>cde-3&|2(?~kJ&UoKExK~(wte0 zzg+iE+4IdG7v^!;whr`$-!~2kEvgg9z1lR9gX`VawBZ$5Gu4m~TfYTy)O<&T(V#AM zWfA#gJh7m*8r82rIumz$sR30Z2Xq~EV@hVzY?gv zQ=&e6y4%giVc4^seWS<(F%d0Xb1^Qo{%8HxDz{DBE+qlWOR ziWJb}<6u7?H=3vVcCW{d9!eukW9q}FaDQK#+s3=AEK=7QE>YAPDtb}^(qD-rscqM4F&7r?=#Z?LEijdhzDgw8Ri53Lx zZ#OkVo3HBNZD96)9f-B!+^0Ww&m?z0+_rA#j7mRuygm|~bl>-4c_ka=-rrwPvH)RXqf>!%1U)H>YSF^Ro!7j|8G6DG>g{!ONG@)lZu0oDFe4; zh@}hp1!>7+rP?x?5&tXA;&kH42_g6O{`#Hp1ozB=->ZxYYp*~b`7TqR5|8aqp%zho zVdLV$VB0exh|(KduWjAoL5?U)WM!qjS4sKCG?Bogfc*<(arY+Q67JyR-&5>tF;43= zi!0ICMOAImSR8`7*IfU34rDWDZ_Y@f=@~n54`O5AiZE%uybKSDVBxi5CE?*Mb2A+_ z^~Dn}Z3~GQyc$j^mEK+4Bvoe8mL?Ij5x&WR4&|Q8(v4I;T$%6fYh!OER80LDc79@i zR{G!pzn8j;;uFf#zmjg)QqY1U<*SJ|V8CtUZ*EFrI+r9!jik4P>g-3q+s1}XminTP z%X8z=Kc^5OnLAH+#?@L)L7`l5&$r(lB*jfb5?|CXz4si=+MM~hNk=wOHH!jgNJG!m zxVb;AlU0fx?Z`u^mLuL;MB1-Rj^Tr?ZOuQN8<|4w1O2 z^yjc)h0l>j_2W?;fr`jf*Or@%S#u2f47SNvu3X=$S11>BHSD>QiB1}EuaEZV(e1Cf zFqf0G$yz(yW&wm817=`S{qS13Z`d{~D7Lra)NO;R)h*)!~e7sqSMrW3nVb0cjRJC60ISiL89b;&Dv;7^*6%n-*IR$pz% z<=o20o+ldD%@GNSrTpEBBz}{)G>f zNq+f(Hi0=VibK(FvK>Uo79EF2nxV=+{T;b;%R-XowVd881;P<0_NZ~B7Z@E87ksUq z0W$+>5Yqf3)@qNP8KPJ8NRra9l$jE;%c%U}LGflPl!MDgN;92>6e zEVHk%Oyh_RD1KCQl(nWiJjrP(<~Duy12L#NUma%1m)6r`Xg{x4jCSOf6KKa|5ZUZWLNyK`%%P|lA_@;CX0)2^%z+n z=>W9sb2gUc3&be*vIrQR^=htgfDRhZpfhD%2c!HX?m@G_MGhmN z8u&o=hMDw8r$24@V6)Q62;OXv`+bv~iSRy}Ad?(ckg!$Atu18gHwRLrDY}=t7!zh_ zYD{yeppKm7&T6#)_T&Z$6FnWFoONtL5bMqiA6}ztruf+^^I+74o2eW{p0H z(0^7@O*JQ7GbirXCf<-kc;KXN zUzld0q-USTizcuNKP3-0j1&P?aOj?3P?M9w`-wjbnJK4+7% z#Py>sr0UlmujNVK*cHC*helSE3PEcp%rJ_dXB0?#z$LQ=Rs7p@zFR{&eGKNpl-=Am zs#cWGl3!%=ygXKw^Sg}=3nI^yn-A1UjecaAexBj^mpV)i_FarsKFuP|QBNNH49im* zrj0#C3q!(LH>!+wcVhw?xHP%}arTNxWXy1fR@+U?lUCkhMP`>(GO`XtX*B(wcWK~( z*CU~TtTRmZM$W%yqyEbbLpg(}tl0nd1m|wZi^@RRHZCys67WS-#@$6t*he2lQ1hiSm z*u0Yd_V1JjTT?UJQ)Mjey%Iwc?CTy=Rs@{$1PlT??3Cjr9=*S+InM61=bw9&f}P!3 zRudD76bYu3>hhV&FvUO5P8hF?^mn}nt5r`x^Mz|+Pm|*WY(P}2WgA0R`um1GqdlO( z7Gdx#f6qhYu$BMop&X5l<_m0d0%F)vZ)6sCfun_A9ON1$K(3Gk+M&-X-Cfri(EMp} z)2>=l(tCPMdtc^(kIdA3?wPB#SKUR1+LP4d`|7)kYkMeqjz>9Sq?&}K;x^JD5d?5v zY^a%O^} z#QXwb70eymv4BcofjIicihrv!dF6x-$IBx}THyC`wBkHUV)lWl|KW(m0xGIwx04m~ zIc9=4H%(i~tJ-14Q1fnw*SHgY8<3)CJZHVFl6gKz1r^m_UYte5_$_j1cz@!FQ+^c` zt`sySU}y9QVb}SXrL2Nwke~bU7H3GT#Mx^z{Kf0UE4EO`YM1qwMZN#Bh-S*M8PCA} zxtS-;>G|$Tz{N^Odl)4nIuc+_ir;cA-58hSp4zp^wW(%??y+mXS8?JY1C)?9QVDlu zZmm5Bo^S)g9{ zPb~Wq2!`P#y;a)4LDBf5;u-|h*HP4}&u)UpB)VUEyFp1VvK)^E0wQALyWNPRab4-- zcc@#-aWUpUsx0v@Iv($o(^U#JUmyOif$8Jg&c~E~$@}sO>Ebc-7~AcbH}ERhN0f+~Z2*OP~CbA=EVf(2VK8_lgk)u_H zwnds1=&bd^*AN&<|vXsa;mF~CK73EG*`R^u@?J6_? z^*=AvaN`Og4teR_T#e_|32!5nZ=jQh`AGpt3N^Ks3!kmfVlVEO#Tj#G?d?uwMG_+4 z|E?k|0sR9m8SSq$DjNjN*|7BoZV9l?0AoIbvaB_xi)F`) z9Hi-g@ye(wU|#?r;<<@e1>yK3Ota$CQD|n~(j_p0a_H#j9yFcJY|}i_EqpNRwGbsj zafXcXFO1#dU_DV0mXy>lCNe)-6l!)*{3m{KBj)Uy0LgYnhPK2~RAmx&Gyx<33w+5u za!aE5q<22Qp!i|u>ZFyAklbat!%SVy0(V*cexJ0&;ICUL`}g;ZZoLxw*D(fW727O; zs$XwS2<114rH2?nBR5_ELbeKVBD34CXW1(0z!05C_sUK8u$kBbhL;1=c0oy#A&f)9 zxZhb+spet++kc&&c=BmfMJF}yF&!jAh;!Z`QTFI~p+jmVV~X9dc|Uzy41pvxZiX#& ze_5(K#E#Ma#cO?TN_Co1|F1PB!whb7P&*rwqXY5xb=ys6RfIZny{^CC6VL!lgBiMx z(V-?&a`BS_LX>mJJw?S;A`fxr@9wNWHRR|G4V$d%l8R20`H5~yHu~Sj*BHS>ZSx8BQLGX|oa28Q;Q*(S|Bn-5^Lj)7 z$265;OAxij-$zlv1cMCLoHn9}aPGew&AVY8B}P$6|NmAq7K`7OG#(iuoflBSyYET)Y zncCH{sZ!tR@`ju$)YNac!{toM)!|{xZmcq^+Lj*R&!378MFU!ZdpSK7r}jI2Y2oL? zaB;iAtbPB_wU%J+|m+FghzOIYxnZk_*Vk2W`f+Bbi(iREGL=-yf|Oq;fV0v zHhSSO7CjA(ER~zjgBT8)&y+dMeH>URoRILm@Nga;=%mJuH7`nB9Z?F__{Y?<7*@Pq zT-&hb@Vt-@0d`(ZV81IE7-%e4$#B{QYfoJ7m|A|R3GCha;4Yb3lL<*GtFOTI+ZP04 z%sjZfmGJL+m1!}_w8r1MW^mI}FA3lQ7q{S9{NJ&duBPLqk}vb(id!PHmn+S0RUx`x zBrtyEHH(92qbjWlz2=MRj`?T;SufWp&%Oe_{-gAnH$DAE3z86_beh=arFMKf; zCwmS3p>sP?=^hSGP*dAU+i?|K?)ZX9sTGYQgM>vs2%h%gVd8N;Et`N zfZTpfAeeL_wUtWpY~a+`Y3iczyvF_FdLyRqGnaVfiOxX=ic0TQD{ut)t-@B-+_cd$ z{?P&iO{|zbs%L(MfaRqU5q{hZ<#pcd)j3D|R)OR6w24*vt@?qtwU0CynI}yXO~mI=?KnACGufW57}(G52dO-v>ci?6lJN z#%dl4gDDMjMbaLh1t-||OokcWyjR}am*lPCJg=G?KNg8q=FrNY+FxJEu&q0?b1Y@{ z2D8NVH-Dw_{BDq@%D7x^_xh``rUFGXB+fKw4KhyzR(`X0Alhd$Y`~r**jUlR6E)cT zIGe2{`j#^d3(wM9Y`$1zM{&TP!`Ih3FHW-(>}$ppT;h>SEqJ<|sYBm-iNQ+W+rh^x zd^m@@eFM2G25$_?ED9-<+0^Y+s)o@#mJo(*pW2A^%G$f@lJBO~^i^CtV* z%_Y$YY>WSKk7&Pmwy-vic;#dj5<|P9nPO}zZ~1Pyma$bz2+Y<7iI$!lgNmgtXwH0N zlj*P7l8FEyic&2pH`%v_Bxh-*b$V7`)?+txVhLfn&4(5Jq=c^4Kr)Mp{~80l;0T?a zg~p?>Cmy^`Pg^TMKedl{Z6sxZ6MiK3?j`gg!|I~hkUQJEv^nvaPdR9D zt~JCs*aJ70_}zBbK3jvWq8U-4z$erWD~dG)b0I%P7n0u`pGq%(YWu*C;nuA(2Tf_+ z3w^uaDBrVsVNZO6l+QS+aGxoHHVt`p`x+hAyrnGFv9@SBx}Wa^7fL?{4^E;I6K~%m zjnd-vw&*_|KKmiBCRGBw7i7&V59Azv7uzVbOZUo4jK6lG+w0J^7ZRdT3h_6 zee?SL#V>lMyqyMT6~b-rW88}(=a?P0L#tAS?($6K5(oH#)!e?v{c*21&9+T?MY7WE zx!f1r=)15qgFsjDWvi6vg*MTn1vveEE!yXE;(&*#x1eY`e}@7jZ(TzhGPNZ9R^ z^M6Zd)ZHP7Q)+ViCVuX(umG(W9;9=x*J0R_^&$24>O867<+#hrV!J^P_hJ22(2^T< zi1-6Ts4zqEJamA)dUTZss6sy1BRW8vJq0E9-t?^QJ-e_WnstiGS5bIx9#o8{Xj_0MraL2!`N0HWS@?a{$aN9WK>;T=fsyl0a zrs=V2u6$cLmd4oa@^A_j`G9tb=8@;UYuH3{H_Scy^~Xy(+Ax!zd_T;jSwLZU9#<45 zTA)KIT13uqa4}(Nwk9P1X~l?R1bEg@A@O#~%Y~?10dVRNUp|pSqhmf@o7i*h{UD_`z>P6Nm z^25n|{C4L>5FpP5ea|;c8aHx#mA%LV%-=-rw#X5~M}@RC?~$;^a_^Nn>gEj23r*>( zTPj8|dyc1%DAU;ojCN}+zI%|VwNQWitoXzCFa#NgT!~PAM~#}9pr5BEVr|Z+ZYZdo z{k5D3U7PeQp?dd2W2Kv23p}Bda=U%uIhzf6hbCWmU9qhy6P<{Fp`2z}4u@_j9%2#! z9N7%6aerewF8;U3s`dtM4*faV$Dbi#;_##jFHRv>IOZ4L zW6sJK$bfU(id)+bb6$}CTaGdkxu#M@g<8DpRF$E2Tk;IQ=!)my>6T)VE}GNw$SrK7 zpo9gE_(Z!oFIPrSNA~vW<5tn64E_u6kP}zjfIEYPeWB3OGlhu$g|8n;t?l4GsLok} zA3W);Zauf=cHxQhNkBRi>-wkMFfGjD6A%8SlbbhVg@HeIzi|Wg+2{S!n_aJ`SMlC4SItHVq4t5eMBBl1%T8oD+&UN5gdl(XKlTgn+KpqvyxzJC# ziTYT2XkyT&lIrOIfutpag0PU_U)zD64fch}G2*9pzWK$+vJhUJ1|n!yk|78+j^H1M zCbzx^m}arvi%N2hcHK7LV}c7XT_!3xdB4|`_>**{ZN;h8O}vYEu_mXf0bn8@N=p1# zI)8#uN+e*S>4=NEN>P+ZC+f`@m(6!l!VxkCYd;qC_LW_P*g493qv;Nh!403^$6|y^ zmv)DUFHgu?VVf+S7IAOExotA7p5Xz(ufOB29pCAEpa37me4NWNdrm0KusF(q4+J(I zE_xODfb7GrrOKIq*lMvqD#PJH5lQH zGNXqvEsu;D)7s(138k8wow!#YB1QvP%S}90>(Ya*6|OBZi|&Q>hEMX(?uvu4iUn|? z?uZ-R-Da$;--|xL!6^>&Nn*iK&V_&i;R|~>jdl9P`i8UO zjvp@qg*56tU)*JIY?W#K`Tg(y?+EOzwcWO`G&sWQl!J)a+ErqWY!4`bT~U|GKz_Jk zk2EZ!4pii5PnxNSVa=7gg>Mv2y+!`i_h z{ndvmq}OhWL>JMj(k*xpZ>Qz96O(Jh`+;T&^geDEK*gnMOH4>{&4M4PHzk{F~w3;Hq8-+J~@@Kh+=8N!hi#qH=o=_x4D`2 zd}K{|${h2ooqqREW=j{z*^e@;kfmurYgi9@26>t!7x~7QTrS>`iHHs4@LwyG`fyp~ zPu-rd824vlK%SOl=Cy#V^t9{`)H(F9=L++1D@s^5{svBKNa(|rC1{k+^Fs|WCMAqS zB}C7Am0ci8=|5uC{hy;2n73cJ?M}owrMssdQGzBw+83u@;0V#JdKQl|kJw2M_|DD; zk=(qfDECcNn$_Is|(Ksq4ZLnnv*gG@@scU1ZH&bJOUQM z3k*(RCi0D)1Odjj&PCoFE3%glo!=rNo1;fIL4Y;?WZb_sMnWsq3@P0e4S0~<7_ z%hJBCfij6`BSydCLN0HrkJ}p~jyA=H*T_)Arwf~J+Ndkz%bPLu?0;Nb?6}?lc1&+S z{WCIwwA`2OsEV?nbbCizsn6$g19s0=8jNL*Ba^463jTEq1K>=bx@<g#KJ5YNaO>7LQ##!lk9Tw$&AK*0v<;fqCe?Bno zJN`*kktQ143xE@cv?PbYD#}XJJYlr+UBjcc1rn=JU^YVFr7BUeoGJ8M=%zb8a&1dU z6}ho~qS_Z^IXehc1Z6<g=`niu>5cS7B;*xa5a2FFD zK>;jHWfsoj-nUf$pH67WH#LY+t(?HK4KD33~UTK z8|QboqnHq*635zAVy^?y+LYJt0$C>RpIAN#Gu(;(Ku;b_)}EzBiAAj6x;d&kJYQi@ z{g#6!n>gsvf*SsGYI%lbv6Q)P|70~UN@Eihw#ID_#5pn%exP67i%&j0B>Uv` z`eTxoQPIzEvqI@n($T%kjIP~{aq65El%~U8U^T5j0?sYxa9k~B@E0|w+p4H&V)Iqp zKt!zJ&RDp!->dfEsJK3NjmEsFJKsExu`wMWes;gPRstgiW}z*6?&!N+U=+$pk$-6& z7xa#(IhXmlLVSt*oNLU4mp@E~=y@IyOx(N>uE^gg-s(R0>*hy;BmeYXHXGhtA+u<7 zvu}}F8QJql@qf14z0LX%1kJewcoVv<`1um{0JSK+@6h}$GT(dmHQ)vd$yBlIkO_VL8w5kSwSc0awn9JiD zoa-Mmo9vA7P*wJ9tXIRuA(qdoK~)O@%`Dgs75-y^cn=x3a?7(Nv|hyq0YrAOE(c~T zjZ0aYYnEniGYh!)zVFxd$Y7$tLdW>KQDr;FuB?C3GXhik6GM3q$V8_kO?T^~)lLh9 z#9sAd5aqFR8cY^do?iAl?iuVj!~hsPN#nWKVvwUUuHVm^`XEvu{^LyYOku_*vgZeP z|M6Z?eqP75Lz0Kf^^6%)%N;X)`Jt1V!J=!-OYh&eB)FwVChJ7~+c{75#Nc|;ZQnbE ziNeAlBg zC)PBFC`a2)Ivgz~J=i_QVv2+BuXILyA**Ziwqh#R`u<1s4Uz~gN`}5Lj3cm0Wl3=7DJZmO$r);i ziF-^7mv0X-2rF{}i;Fa{{p;AzH6H(L*5eeMd`D7go$_&J-YrXwBXVp!dB>sCOsAG) zIhTD;jqY4-$mAUJKvLeV?H!6J+%a~cvsW`kCVNRA-xTv?`8WsKIEkmzA6+vQ`xX*( znd1^|yN;Hr_^VrHBdhk$XxFJ~yXjm)4Zjj77Ak|J7o;dsBLzpJMpId(!w<{zg2p2+ zX8mdxDJsdZU4hx_3BaJf{ZCkWrSEQn_=Rl01+a1Cw-SfzlUx|wm1IGd!cWHHQ0lu44iefJlBI9Fpp!%)y>O`G|nKs&E8 zZAg@W{U#H8;GcYk%Zm2N9!7G6_2cesZ$niy#-YD{Jo9`ZI4M*o!+w2=xui29B|dMd zT-5xDd@#k^_&Y=A=RZFBeYO}qEdK@}mU7@m?U8KP4E1~YPEAON71@qu|4Qut@#8fl z9YcV5M;z8WpYl)#L;#40!7sCfFaV(qEo1%EeK&tqc_9 z75QL<93<6gr7txRG<2GC7UoiO|2Ff?iBy_tEZtN?4$m2^Vp3!LM7HxHSwd5)6_1k` ztMB=)3lpceP>#qgXpcKY9y)0sZCX7>vcg1O`?>`WZRcBFJrZFOBoSGe)!b=}yMG$` zdtr8Nx~|bP*jMAb&M?pV{iw&hu@7d0QS{y6N{rp<*yR=B)tmhi8O#whvMy_CG7}_6S*)8#ir0!B84$PQJvB-RDn`q|F>9=z&(>3I#&2dqQKtpmFw@F57(ht=BlRP2$>RP>5uY9}=)8GOI1mAVnE$B$182ao zm$(AqViQ6(AL{=W_kCEoj~5unNo5b5qzv?` zoWX3Gyh<7nQ|1A@aPmsQ639~#Mh=a=f_*4JurZt#!&x?a>C5c*ltWK|#|8UJ7${zK zzrINKq+Usrm=N5O&X4a<*`2g4p{6x?iSRs zE6b&^uE2ew%K%ZOqhq-@6`vp7u!~}Z&Yo+STWg0X@Fs~UicXIMZ`kjQceYpmuolJFXT8lg2nycJO02bZ(4qCH=h=?mG zum&G}0z;Bn++0;G8hCVtiu4p0vSp3E*5E#q$MbRU(;@D9ppwA|3%Z)NGl}-SoVkB6 zt|tNgsM9BIcD^@O#)`Ge#cdoha=H#}vO7OogO;kUaLQ8Ii?@|+ApTe@2EvtDKefN0LjS8m8@X*ztF6hc zjDq;Ct;>sV7C<>2fwjVnzx{(S)0_$pED&XjgADl4E7j$d7>HjGaalejPgP#r+)`3} zv#7A5IQVzcuRYjKl0s_20%)#uTGC1r={8aJmR8dA)Xl*E0NI(ECQ#?h55SH+#ZZVE z11K)Uf9P=xE$8*s?8KCy`_V-F%fQ3N{LCbSWvqL!xu8l7>#;)bCiOO=>eQ`hwp4L@ z0i$ouPEMl`67fq?K+|8^8y?b(zzi{x(-s%UnS-TKX>Sfgv7Y8=% zXo=wdS3Wi>VM(X6t$?Qxc{{UP-H}`7@k2w8^u&wWkQKJ=$j^dHKanmy$SupKq}PjC z$L?+gyq>GFZEDPx~wK7*P?rNIpYUdt`l?u2>&%VJl#WU60crtng zWFJRkLPr$XW8!1`Z2{R^|M)i|2g))viqSFdRE%J;Id||>H}`+~@)Nu{;DH&d^Ssvt z!3nCE*W}Nx2-|QvTONV;b+Hr_ZoE+Dej0WgWCIZVBfdIduOI&95nMI=essF6A5IpR zaN?7{taGb7*#1hDj3n~`pIzY$zuDl!k1nrMfoH*`RFb)=0OcVa_-7Nt`rC?|SI2jz zy~k#jJOJc6T+#Q{!72Sa582)H9VvAAVU>4vB9?rQZ-q&_e_j*k8I65@*rhya!KhE^Oij&G&o4u};6CpFpLZFw&tuo6slqsVN;|5uzJfYC#_TAHtohnO?* zA}cs|^dHJ}E%u$y^`zTbE*Cie&}<=BW*&+s4XzP!dO9Hyt$dg1RsdhJjF}0*E35T%ER4v}ERbTY^<-QL|2sK}!}Ou@Whux2iG z_*pD7FMBLXFyh4zr=HO-;~P61F%Eyy%2`36=*okBi&eLPl^T+U5S0L+%tH(O&cb~t zy~_)>mA%*MfZ6Or&wA>5_J-PQU+$ZU@e`HYAigpD%}EvtAqGLTWWF2I+y$YE$|F%! zp(d%NvwF3oV((tW`1uoO7N6zuZ9DBf`^g}Fe)~OtAf1$pbrcMA#h4xf)u?4>`VQ7& zrtVPAo66Q;sY6xoP2JBEXD31l({Vxq_Sl9On;oiDx4bgtds zas3QA{N$Q*J^_CZ(Wuj;l$@yYxc7SC;rh!jkagW*_L~oA8K89gO0u9|Wz4bKzv@b6Ik^9RW_;bDfQ4$E^w)bN~ceEPzZaetI|VALz)ZHM-k zKvUaEi!n>1ghOi$8~&Xl#pJT8h0i`^fa95Q@W+t`GvY{92m!R^?A@Vw^?O;$)`jwn zd(+=ut^0qm?X6G5AcHB}AD2N!na zS=U^ENI$&!{4KYIs+z$`@dp4kPafWO{VFN&W4BDD!aUmJ&tZ>ziBC@mzTSIabVe+> zY1f}6Lq=KI1S3<8QlupoWVbjZtGPY*|FHL#QBl8N|F23aIfCQ>0!m1SFmw%|f*_)F zh)8!!42aYqCDJm0G)Q-MgD@aNcMdSpB@A&se1G@7zW4vEbJlru9-QDIwbtje=i2+a zUVFda*Z%M;sMt&sYRAAi<$4B>N`Rbjw^spKpld zWI$ZPc!$VWAC=!P6|%Q}?NSpOPL5f&Wa;fL3psAbeMCb1_HgqE@o5YxrEO0!s5xv_Q-`EiVj z%@ONOGvn>Zbm+9<)vq$n<_GfuN3HG0 z4jI1}@hmkM4wfDxi9qON>hv{}QpxZg*^OsYWq+qcH3u}ahNf6_A-Zf^uhDixUaF{N zSLSsqX0jRq0)d3QDj$Uk%K*hq0fdaSH$`MXc<8iali2F2>6*$T0bKN#H31F%?Dg<`-)_%2yzq`UiKTLNWMdUWDUgL!DvjXe6s7^$bCiJB zdzM}{!h6Vpp$Plgw}<(6PM=IXp}h5ni?WJP@ZcEajS=tv+xH|fFQ~LFa9)@jEnVR~ z1mz(SDbxLB=eadyX_yo8#JXS>2wZ^MN_DwFVOzR&>@bw)7!3>op)6me7pih-iIq%Z zhv=f)1Ys%-Lxa*@$Z=sny^VdBU<5WUGV`Hjj3qwML88HySqKjawE+27|vpqycb5dD2*55vH zR-J9@vSKP@)L{2XAkdUuxrlmLUA{84sK|3F(zBe-s~d$^oK%8j(*wfje?4=v0t2Kh zxZl&Z*9Fa)%z!YywsV;scQ?dSwEw^+o}mvQ<|o?-OARM{RznQF833HQANvxdasOtb00PfZ)NG9MNpOSQ|Yx?UNP zCN=DlK}lwz?>hepVW6;Q;NCJZo{6gJN9B7x`0PXVKA%H5&98;qO^+uhvmL8d0tMuw z)*zGXlE1h4aM6!kyBqVKDpU0g@(B@)fL9#l$)R`yuQRC1+Rxq5X5=Iawn?AxD8{Mg zFo42JK>t*uKg^N<_=RfCrp>_PCxlTqTWEvsL_0w9I31{JGBsHaP&kNov1p8PH1D`& z>}Ms?$Ok6$?j|M9z|M)kN3^kKTN-GeMKfR9nFiLl9cM)bR&Z;cPUg(zcGjL2&eNPo z?$0Un+#9IKOZJWo&LU?42EVY8`=6VQN_mdIn7uvpcR!^ti`v*%Vnpy=&pgdc)nsbF zu)U~>Qp+V&aE=O!AcIp}|60`@bngIFrR?k)O!&Lax`bb))tkpWDb!*UNXmD^D(Ux)+jS z=QH8@u8jO-fZ>Uz;>?esansG$FkFJHCj zA9g2YAvpAtUv(lLBe;F_n?AV%!Dd?}FRqqyd-CT%1=<#-XB+~wE?xOA3_{!}(ATd} z=aTZOh%<*it=s8+)h0Odai#~y{(_5*cJyK3ke-OI85iyKy;bGsFXw&li?R4^XCOt+ zBIa;VZoVODs`t_{#cow4&PXQ*^1cew9PUD_-N!knq?&)dtUq#R;&&d|1i zkY9Ls^J>VAA&*H*+B|juXgjcy9CLDWsVkuBmFVfQV0Q<552LbELsiswNEPj>7y(O# zJ3}6qaOzoJ{vZb!qcxvDuaIjK!C~QZynUJ8Maa#uc>E%3xjMKT&Py9mXa zF8gj%9s^zjO0d&+0mWZZcDy_4rL|KoIF#bNi~B1GdkYMkHe%I9PhUUaSOGX;H{@9d zu3CdB18=Niu6N-KTIa3!Kf6M=2cX@q2Raj12v_lvba`FZ!`V}y@C@(h<;M5O9_5~N zxeOzNxdZdkj8Jq%AH{@EIbj|LFM-$Wscu16dB4;7{f0(-9|Hb6} z|61~~+4ZH`4Kx0}?;HCUGaQ^7kIo(KQ(<~hdu}y)nRLmkdQV`)I}GTzZcuv!ZVx33 z*VVq?dh0v(HOcMr&vLxJ!(o_m_{h9puT4plmVx6)SCpj6F+tU7;E~Ht6j_uh?huO>RD|UHa%ZGe)+(s z&e(N7yX&Qk1L^C^dF8|1#sUbsvG1i-@EvTO(-63&_P#;K8dvH;7?)g~hOfC;|9*-r z!`kI?7+?Ic#SQw7BbWc=1WxVfEgiw=>esZMX~bD-%r{UO1?%X5D{ z+2MRus!xGT%79U*z1rTfa_R=Xv;Lo;_pZU3a76DBmg!|)$^Qs}$H8~RJWyk4Wz z9M4MMAqh!e4H#`kN1fW&8%II564~TB%c9a8%{9OKO`A=;2ligkITY%=hI;nOXW6yb zRPO=d>;25@gMxZCZV$Stzt_g@mvW}CKho#>M%dVaV6G1*SDU_ON$xl#91YjlD7v+O zn50hao?KcXPukfZ>hHxMrPP`Qfd%$$t}PCSEfP#m>c`imH}4|x-?CR}%;5V$VE>QH z<#O>mZ!j(b3bsXH6v!mX4crH-Dtr!7aV_veZd5V8Z{2?qG9t5{qb-~orHtOB-mvu; z$=81`qo`uJ*671Nm*`dfT*6iVF4{Sc26q;uV2ogjUOHEHfBXxk6 z$n;;SH}wy6p-rHJSoCbN1LsbIgIpQr)Sb_-1rP>Ed)^-fa!;f*$E)R{#Nu3R6%V_3~Sof2Ja z;>zQFL)2N6U5oSq1yDA{m*IybY6T%MKBLCkTHlk1uBI7qLV}Ss)#oV5bDho7Lc@k? zSNmrJzP;EW>T!{5!;1AYL6V}gIae8ysY)vsPC`eBQ%=(g$%PC*zfSysrn=0W4k;P0 z-WxC=-%tUy3j`ZxO%&;CuMfHt!ui$v6I_UNYkXlz1ZH>B^PrT>{I)W-3#5@9Xfh%u z0~F<;g7AN1ak~cl0{Q{wSs9Xhez*=YD4>E}+Y>Xxe=v*U5Gy zOjj%AL2C!SLZ$j_L0al{E@xdLj*U=&zH$M#aN+u-&O) zlB^u+{bT)zepC*8j|FE5@KJHzva>%{+o+@oLYqp4g_pjXL%e>%Z+E6L6}DVT zOGtrR!!ST=p~lT1cGcsAgLq$W8hrNdRHn2vn(TcjY^5^0_3ly6XZVVrOQVzUCJSxO zig=0I5GfpWg6=ma}YJK=g`(#;ED2dJiv%c&%UHaOc{;W&Y!< zTKAW5cM7fcBti39K_hH$z3zT0JDNd6{0*l^eZp($#eu<{%msvxb=}ZAwOiMSWrzYt zf}>Q=`Q@DN4# zwZxsz%}b3X@{D0#$hM)3n&&V7&3F^y!dJekLcm1?9LVaiOXfFu^Dh_s4 zBWEyC*FGRqUV9afXvcvz0q)o#rwjpwLJ0xe4-KY3@|Ij=V|_S2w{dghuTcaMMb&CR zg({==%VZ;7g~i)yl%)6loEqVhd-NKni;ZQ2H$x|pQk#IOjL+kiQ1ddTK)fL`PbV<{ z;g-#SuqrQQKrd+1sW22@gH7A0>hhz2OEb4uAjgu<&PSEPNB}J>8HMYv1Lp;51RpNp z`=!EAbnJ}oQKh~H5wL=*3De`GbO1fH;993Zns2wR8EyG5I`A*f9e@*eKXX9|^dz*0 zV1gC;1@vQ6#0Pz+n8@XpJvLgsYn22*>ZXL)iVgC!S4v7_HnuAwpzo^`_XH_2Q?~rZ ze8k5FXjXefOipcvjsb{7nKjKg_J|=P1Lz8cmhPX{>rRM{>w4aA%Z}!xteP#!QU*sy zirXzy|BUzM0A|#ff->UqMXoGl3baaice!w>Jnz@xvV6H%|CnSMhK+)A45)r!Mog0R zByKIqHKG4Xf*m-@CE9$O@?h_lPVi}LV)L9ACSCT28g_UGW>bq01TK7~;oeDR<6)Vj zF5fWi*Gv0TE&*tRl6cW*{R3c(iy`UX7H$0fs>$oCS?aJbzNTLe_k-+BAvx7NLwM1W z6g93Yrk*sLxM1pfygM3rP<29`moL55O7&ObkgHS$bfz_ zL8F{vP6CB|N?s8bwEa3-irdFEuu{e2;ymyXSP z^F@R#DPFzezU9Yziabc-cp(wt)h(N1t^`eFL9B!;m$fHV8EmowN5}6*M|Q2>$rmo3 z{E>(?L{js@F_@gmN>~-5)#RJCRv!+)-QV2pgPww_I6r4aXDv)t(#C2^ZFrAAnbiz7 zG+h|3iS-EunU)=$YHp8KjKOdQex;F_x!aN!r!naYj%~@CpzaZn8`}03AwD$qYDJ zdwxNQ=|8&DGsD7>e<-p~>Cz2ig?l0<9qE%1T>k;NJAsG~RBrdtF32VG})EOPF1+j=z=weE&Za6zXaja>dMp%%N~jVBX{$fZ3$ni2f4 zNEW36)b*1ySE4(h+il1Drb!@K=*b@%eker*zJL=sD`k~|_Ek+PE}VG)n7njX|9fbQ zxK8nS`@{nPRrj4fcyC3ukh1mC3WA36Y{}CL^>`hJ+d`nMAy8%*FdPe}KSn6wgfD!z=7PcjRj2DoWK9)Qhu?qBho^<=F>MMJ7Caqk zda_0etS3zUO9ukumPwh1e>ic-B2?z!+!Tu@N0fQ`CY_ZOtDqCXQi0MX*qgQHePkEF z5PwRt0|sgly=IM~7|S1@jCdUAU&nY?dol-pWd9$;tuYVKHkU<|!QZw{kuoHb>rD>o z3ZvuqKjRPK$=@Lu`@NDEs7BD!{+GW#6`)syRxaL}ET(LZdbz5;-gm?7{F)%j$k~N@(QDi6@Kkh&8H|##v;i=07+Z8YsHD*$Zln!DU zD2CnhYKukhjSCSJHgbFE;r$NDh_7!KbKa@G1`6MB|Q!yUV`!zP}bO zW&ZnGK1?_Z{joNr*8b?pl1?EzE{QNg`_HJZlCGM)WUZp?8PPx42uKb6nO*)GAP?cs-@oJDNlw??@NI!HQ zw!zP=dtAW-DT((ZT;{ldM!3bT18gxE+Y{Q;S83u1k~+%bDb?Xqww8Hk~DZz3^iaGwcw^y_4MoRRz7_FLA{@MbgZlkL=)w;MqR z_)uQtoogv;pp=dL_pBj+fZ&bfWvtG#YOO(;t%c%?JMW z>}%GO=SjcGa#Vu^%eX0cmdO>uaTLEkd-M561`C9rHQ?dl!yli1{m}UO9RI;iQ?Fyv zu37!39}<#8zJR2BV%mG##cKgkv3vNYNZjIdYs{u-r)I*}@A718Zh%5vQcUbDIaiGl znB(`Fc+%{-wHYI^Q(K{yntU8v0r;u)0e)7$=T^v%Fo3yNO+kQX`mDB*rV+vL@(bRh&p)Ayf8H3B3yMm9ISEOoB^%` zfxLiz*iDZc*HNOZBC#^Fyms5mW$OWZ`=m%xgPAPI32@NMKh%AHsxv#RNI*rTIPFiMP%@Y!Z#uXMx}YTQ+9pTQ(#s)Dlm$qHP3*#1uu+U%s{iZ=V_9E zIBHRu2vq@|;V{WPg;a6fP&5B;?GCT{W9 zGDMJPuNk|$oAL#oIf!a&*lD(iI~;JWPj)zaj{a)9(A+do;fC2cZmmmd56L{pWLl8) zTu-vl-ewo}i~6 z>eIQ2F}a0G(T6X3k6nsBElxIH9>eAOO}?;TX(7*~oOboAZMXZc)&j_yHKrYynU242 zRvil~vxvtI);SgTK&R7SCsR8#SMjer>eLqNLn9sSX8m;Cd93qyLm0h2t_ey09ac!$ zjx9X(wMlG$=GR(neZLQTWI?)pb)x9_eqo{EqJQT+1Z@8YP)SXHlB}Z1E(DDIR-R%T z%qP{9Vud20r8$p>O8X)DdYcOg6DGxodZqewreg`X?sX%>P#r81`- zE~yqeo+PC<6Jc|8{e+yzA3(m!w^2TssLXS)O*Uop{D6Hi?MZDgnZ2I^Qj^-&H=J%n z`hud(e*F8<231YXpZb$+vw_X`YaAAGU3$$GA>@KZ@YPw8rNlw#bk{gR}&6Zv9!6MT?W|=?_%Dy@0HLRwSI^lU}2l2Csci_KJPUbyiL4E$^~CJWk*dp$tj38K|7FMN#;v z2=sezx*3a|{H4%7iQ){wbPl0m+Gis@jF|rCv0t0J(DmT+JDVU>1$=`=T#2cOIR*1r zozt4Ol5@ABnCk~m|D|Ni)l$l?5va<{_|Z4rgdU-iof7_SPh8V!pK+HAx6AR2csa7) zpiKUPg;g9$5#LKGZ92hKh?7o70 z2$g2uc};*~9?{)NtJFg+Cp!75W(P+3UVPd)cigFE&x!gL73BDE`C$MRSj7;^Z9;?s zF1Ul{of(M7DUd(t&>%ISm}qYpZqL<3sM_U6{cxHx_PgQucSWm6UkTK zqlxAbnEAWxMylR-xI@Ad0q{j3eXu=M<<^VKKv54Q(y;km2BSX%S>ijyRv z^*!+&mj`uJ_hF8V=fv*n3?}-XWG(~oMTTM%2HtZ&P2tYbk(P7{P-+eJtN7@TP)RFj z_8GlK*pC$<9JwRVIR<1besEm>F%wBK$b0zC6!aICK}GKC!)qLsclgVTu`N*~?HDmx zo^5v-4-7vN!eWjjV=Ru7B`xZ-r{T>Af5=wMLD>i~v3a*5zApI+aJ7oLUn-F#ddgU7 zx=!Q|)yI;gOVu`|hog*ta>JoWeU#`cHds@RH^G@uNU_hlG^b6f)glYafIL@GS@?gd zZ!nZ+hUscQnJqU>U}peqA*?|qf(Y)A@$3A;+U5EiBKa7ySu=O#h3lWExBY45oikXT zYe=hZm^hLIEx$+XFRP5eL!6vTRw)cLIZ8c-I~{zmw$7P8ewiLz{|i1c%hwq_@T3Ux z3F7?N>9383z7py%^4)eX^Y^4_Lc${hzYM|0P?Nme^fBYmT7HJISr#9bH_vdUBj-)CIkcyuT^M#7r%dn5A$eIG{B@vgh9#l8G!sAxyEIx-nrTAl0bklUXl*vE zj7&PAg6LL=gy!d-SRDAAu4eIVxq&iU6bwJ;lN`x!P}JjJzaBP4>X3m(;%)DEu2kg;q@A+}ZG4dEbt&7Yovy;Y%jnw;siU@p{W9C#zw?0)8 zcZY9^dJe_Q^G@iHI;MDB;gw8u*&N4nuoLCS4M%2;E{viz(N;Lc^dE~4>;4pgP5{Y| zUB$VYSYwSr$0xk@x~%HAKKsf{HlBE=YmHksIQ?RHlQNN^z+9E%^s>}e7P6$o5SmRTW?>5{0_-W zS%r_h-HmEms=dFhcIG6DqsF*dmB%v>3Ss;yli_Kgp2P5(PnD0G8fST~5~aPBM_C*C zHx^h0V8zIYitrc&A@M_ZPdE&=Xx!qwx8FES z?TDXTgM}f)yVqCnQh1wQVCd<@X>d^#j|C4utN#Yy+6R2RJN`(*u=}_|n~gJ&U^wp& zyE>UpavavWok=e!PkW)m*K}2;{*GT?s=iDMATtH*q=UzgRXUGYnP&XVf|aL1-ABpI zXj^If!P}N3?lWCIo32n1Kf3YQq?X={yzCBWXeLGEOicD3G4JoXV zoS!r6q3efw;74)!BL%+aOm-#~cEBLWBDQ+HoX}Fv;<&$@ob-q$vQjBhYc8d=?fJiM#46G^(6^$JVFmL%U_w^-cUT_8xbz-7*^ zn1l*yyh2Alhrq)9MXI5yoRW{wL(l=KgQsDgNT6TAIsTu9R8Wjuy4Ozux5Nn!DD;F) z&nm(Usxk9bD_AQ`FYtNY{Vi_C9Z0eI#!~6x1ciNyv<1-%jW8T-CBoYT_-Ns9P?c&|mj*Axm1>*mgK>U4JsU&wG@2qsJE|O$dXa}& zq4i%yUZRwU`6DWDErf;`M-zo$R5Z5l`k6J{u9Pp#g3REF1*|y=j*JKFzY|Xvml3Cz z&|hEkvLpK8oNAs_j{=V8cgZIeZABhA>y~sQf_%uu>KHra?`nQ{j(ocGY{ETADq%Z0 zk6QDSN}>^BtQklsqkkwiu4f%m_|^!vQbjlq1#Y#T1)CeAVl${Vb}RABa<9iTM5^D6 zOCfP>gDePE4F^?W1X{}Mc2XJctJ%4CvMjg&mD=MHourLvm z#r##w0;GB%W`a`iSkz8SDR&gPg{C`xsW)&w2wr5-poLxc;DV!K_;WSS-o{h$g|I!ixt2VvabdiKkDFe}Ewk~yLNm$ZOh zm>&2yl(OQ-+Z<@KE7emaxo`3rb z)u#JU(q(Y1-2g;WDWi$!=?59T7Xk&M$9xbe)#bz!H14%q6U9xgbwx~RXQb?KJ9%CD zAh|j=`Jl|KI1Fj)RBI|!#+4(;&XRhPZGbDa`N!fDt67T6&ROO_oAk5flV;CAF_w56 zrN)OJpK&dSsdh%7@pgU9mwq;qJr_>(ve>U-w$@0#8j62r#ACgAh!Q~F{=+yEo>J~e z@y^&sjWLx>SJ>vCz!0um9j2pkfBiImXTs|H*%B_Tq4G#th^YL>Y@zSyDI(>uHS>h3 z;oikBhwbr`1PehlH-TAVSBwQW`fA_CY_;L>?t0Bnhw1N!LWD2GCU2+F%5wL;;}%i@ z)`Q5u>miuJtuMPHa)X8jwu?QZ&xwj>T@N1@ddoi~>PO4KC%4zhY{uG8hki;Ffg)An zv<@Y8R*h6PSb8&-ak7UjihRS*Q0yM~0u3>XYcWZ(a&Y==vn5FJob?W!wZ=?#la_kp zKsZO{AE+pc)_MPJ=wmm=w%%^!`lA=^3-Bvj)Jv$iMwnL_ABz-Bb0DPmJzpqs(BTla zXk*zSXHx0SD#O61OvEO}mT=jtYw392fyuK)KxPSWg&p5<;pBZmPWwms**gq;cb0wP zdw$hghEd^U%I3i8OYeB1JIwc!B6Y`Uw~ed23Z2Q>vl0sGPTGDb;7pBG!gkoXxT&-J29O!)`6~EOsoMGojbk_S*ri zWMO6wX91%QZl}%E_@uH}N2VE!wGo5%zVsgEYyfkRgQoQVd+~^5q%S!x?g_*dxqo;u z+4Jm7_@Dx0gV#f9GJL0>wmvQWsH**q77OE=!xamqV_`91X~cqdB@a@VP!b=S8TQR! zub8&rSxgk-;CG#u?yg?(+Nr@;_LL&(2rqL;bRT4%V00SosdNthjs1~2AeSs%8wngg zilLYCxI-){e)t~dQWOxs1&MZy8IT^dlB4WG($5DadQB}}hMRrgE+p8>V5CeUnD+Y4 zQ}xr~om_Fc3uv=MTY(CAhDo@dY37eSJ9uz4;=^P03~lJ`Lunp)u4qbKZfIZrq31l$ z!i>q!^Gc66GuSGFYuasZQhdw&LqTyKmU7YbaT+C!8FCUkPZK(Z(c4K{gO&O@>ye>a zFC=s>lTL-qz^W1EFY37QSt^gYD9UvbhvrW8t65mQiL4_`0ERI#488gK00JDFfulcE zfdT}4MJxXJumAhzUF3NlIAw=BFTC-2eZN|8IC zy%CiOPMCS-)TA%d)y8UT0BLcI#8Lvf{pR*tFAqMZiyFVQKiFon@xB7tHyZ0+X+k!f z=sxzet>O}&b%u+4q^gnEX14t`;KtW5BUCnPz8{9&!5lOJNMWKu>+jO_Ro=iMa`!U9 zYhpImt#uLS`g%_Ky0)B2PG$NAaWqK(POvC?HS`adR9{;PVM=Pmq|GubijC)+#BX1;_VjFE7WSDX;euJ`t!=!w~moIU=mW?$#IxnW-QzAB1HxBhyFfT z3{g$(zWvjqC@F#s67PR=ZTZ2W@ode(8CKhHZp_Je_1%A773@cSIR{RL&;wT-?T^1AyJ zovDt0{}9`z-#JsYc8|(ZeGhnr|`e`B^O%*w6YJ)LL48~yDwaxo+{ZT z&UYT=-9X5(1lG|GM+8U&y3re;`L?4DxYuI7&upQ|=OA;U*(c3&y+)1Z;;6Vmf_*d3(4T2d z#N4fzE+Cr@@PW#{sAU^c@oufGF191~Z!0rv>4S|;NTZN)APcdzjHcJ#FRX96uhz?a#0=dE&o*H=M)!@}P}C`?pFrz%L1h#4~ze zEP5fW=3+Yt7i5XYHn3x90gTI04H6@W%c{IJ-B6gZQ&(J!GUHUUwQ=HG0^(QeXI?Yg zHR`}TE>8V3aCI!_xml|0jDVy6WQiT`@bw1YB;zkGYYIHR++q8v@+uYVxoD)}^JKg; z(EG=dqR-g3j0CsqQg$gb;!y0z-=!5Vhdmscwom4*&4^H`7FN&r1RMp^EpOF@ z@7jnfQ#${(cz^}NlOaN{MmsbtNnX~Pl=5Jy{dqyIH^_>E0T(>)EW_Fs zirYCQ%aPzoBM>RHcQC-GF1=TREt*D(v5MU4p=(RAIh&q{+C+e^dd_Zvfm>gWm<#$% z6asPQnU`lFj4;$^9Nwls4GnRj{KOD&();A?(KNA2EPXDSf16aSrzH+lCsAtx{y;kI z<8M*z0~|M`04!3w?*=T|3ts`u9rpWQeJD9_xh8d+6k-M0FiY(qy>5Sz z%IC8`U|8q)7age+7iIHFz^+H?oxW=}(9R z>nyu$f|V~8kAkf(iSOU$v$LF*$c4S6SIe=0L!=zPGLyX_uT`F!&u z>%D$SgPER_Yvj%~p+KPWv@rd)=9d$JH3QfE$0@Lc6A7XrIq&;t^zQ+4|B*2YWWH|f zXjENy{ZO9D@4AK8sFPz@vDLh|EMhe3leG4_GUb@(r(?=37nc-&n?=3;OStk?{3RZC z)higEHd1USa3@i3Pe4j!LIl4*4;YhdS*zUDMcSQnRO`)4OvtBkMx~eJGMG!{lK1%V zwnT{q+n9d3Z9XO%zNufdmF&&BFk*~VidKP&a1~W{3uffmJ>ya+P_(O7Aj8huL>>-_ zd-I_U9ZfuljjN$U_RVMUa|jW{ON1CgI)fcD=R^r7Z=C`h5Ma^(>J>Lpd=nrDiX_)i z5rf#@(9IHw{qP?+k++NwF4lxVT+>0Al2NHOEZ z*exrf~Ap!*QE+L?Ft6YG8-R z?8{Q!KxwxWZ?iC=R7;j%S|?5p2qJ8&G^_RKsf%AF-^kNQWW%y4<_A@wYC+DQZ-Qhv=93^6Z)|3i zyIYJLaVi0Mw8y%xg@;H#H=b=uU(}WJPl#sBr84j6vEzM*;GN$s_+RqtNWGMcg zyj!y7Fr}TFa++;HrDUc9hYJQ5W+~#wh$%Mg`FX;i-D3C+jG7Ck=USL^C4P1KNl^5l z!v7%+bux!NvAWK@E$FerT%byCD#dG=V%7s8xA!8G1)nc!N6>UC2PGWTT5pSzpJ}XIh zcZ|jNXUnOUDm0oCPO%lQXm+3N)8^iVzgpUh*QYZ5@{oHsAdhND_3rj(LOfjlp4gEb z$a*Ci6uM5wOlp?X6alI-o#s7#rIc+$PvatGI!y%9m@s20LJ4^u?udDwN!mRaDKE02 zr)RacIIcM2+$u|rl!10y<8!>AF*xh(IWay9hjU`MHGcdSSsw1$H~$eXc}(fj&z#bn z7!X2qe!XGW>rQv93Q{G$r#(p$C%<22~Sadh}668p=n3?aybCF$o z$__L+%U)93nur#0)4x}DG~jH_yYL{((0lH94lRJ>#Gk8;pCt669Ubs5l-C%Jz{G=H^+>1g%^eqscTBk{bhD<)3>y1947Aka+I#4^> zwcE|IhdaD$2!Pto#VUYFnj1Bh9yTH5?`8k!t3nkll4g@>5Rrld&aa%KAWWPssr5lZ zie3om&c{Rd*OppW8J+syX&K^7bUP(|25zZd2ni3+w+}TydEQ1BGNz!XS*(-JXKP+q zCozI_)y_V=@YW6iT4u1+$JQa!d1|{2AXcCqYLx|_- z-2`(GRk9L>F57N~SX2PGRh?Hu`cx+VHOhQA^z_>QD!!gmkq0!~jOE((wq*b+Y$(K8 z((ToR405BM41YT4$ST17sp`s382ElrA#2F!Vxtcezh!H;MR8qWGLVDwS~L%cd&BR& zqav`H?0aR1+SGNTTl${gzC=^O2JBL4J@_< z_T2pY7R}!~yDs=E#CwVq zYq<_LlqU@XyE-jrsuB<6U`Wr?&vsUwN?*=$NjA2V_RDC(slSs?`TfV(mC49_n}_ug zncsTBEe~kni_6dA^r0`)9wHuF*Hxd>EY?D~%tN0gKDIVdojIOUu){GewovtU@-a+U zqZ~toZ2AKhIF^ZH<5dxgsQBF)_MfrOc&^nk7XJBPxYuLyYr&&6f2pCL1TM=1PgqjC zf6Pt({Y4eRFW$`D*2U+S;P3cmv7_EXQuPLBAAh+tT&X1ikMqmrOZ5-ib3c6eK{D!}l|k1b zc|7RhWRyl8J^1q9v}UalR$_wvmWaJz`nl={r_ zsZUO1Za)p4e9lsz^c#``B&`8#%v!9K*2ZI=oASi^ zLNPFpvNYcU>c*55oLrBy_4#D*&F1~{4=gfuyUsyoIlkYhdL(x5AJ~8V;nV++0WBl{ zd4lmp;_sv}ot{{VoMXE{+%pyLU`=(bR)WzqaR7}WeUpihDX=oK(q5-YBAsE`c`qyL zZ3zxE7DDGEiUNSP(~#k{8s`Mlen@QzmJR6=$%R#!b|Ew;A3IGte{tkWyvnD@bApsI zFC_yi4`w$OsHDGK?vYPR9+Fp^Rl2IeqBix1%bT%A4nRbn{~Hk&>r>b@0dOR)t2X<4 zs2|FGzl%`7_{66zCLT}kZ~L#GosE3G&puKW!YS10(scg>@wqi$oZau~d-r9UMk6`S zg0{>2$>V4eX(X5Rih-j;xG%+J7ICMnmJM5qB(&l@5079ZIkrveAvs`;W2YX78%b(q z!0sm$C!xkfN{S`tLTR2TZ`=fl^3##0(l!=%iCaTurb`#fy$pAngQr4ydf>f4|Fy7; zDY;Lg({#Fcm(lt^HO7XxOmi#K)!CxT&CWD*IXdbpAmy<*orXj~+GFSLTOclPp?BmG z40;?-)O@Xs@X3w1O5w} zd23e?zn!QP!0L3@Kk6oGJZt9J<~h1T!wM3w%&Ec$goI!*KCA3JmqCZ7-l*-b!z^T{ zeaCzBy|G>4;4QJ;D@^Q7^LxQ0fN>cdA@(z1FHrRMOJoQ7%!@!c_*P zx7~e0jat^@$PM&*{{a!_#woGZxyicqMs1}tx~&a9{G7I8i?00B=&;mLTf17-!_K-V zxr28mdhhl4zm*yA?kVG4XjAPBUU)-X1)Op(m{z(>?LgWuj8Bq*bAn?n>+h+&g7mcFd#WA0)I_G!Z0KE=SfNi}<@o;m zxZq^`ZB=nFXF6SSQOBF%s|ypC4{y=U7dy)<_`T1lNs2}v4!`Jb{q&;KL?ksg`(d9% z^cr#Ks}flw%`m;*ABI)~*?D*@+m(v8pQ;P|xV%*v-Z3Hg9$r~%=%Y=d3o9ke_7i-+ z3f^XBNH+I7*PWkY=R+P@-OQz~A+4NL&#-x+uho)0JI^qudSdM8p@(F@yzD0yI6+m! zuDDtpY()mS@5f!9?D?4aNtyYm!jN0QL4jT{FMcdR@q^3gf7K~lsKN>ni2duP?`Si4 zDnwI)B;&pUC|qE1lB7spoG(_$RYu`|EARUMna<;QN#T%NIz*DW%sM-|v`8pJ58vau zFtsDbRwHuyf~kjd3{f7sC0}kf`BF1wv!(MbmoG!U6~j8CSqO3cN>IJ0T8NBS)hk<0{59tk-j&DYKVnyW>$o^GlpBT#uA+|t2m zHr}}KRZvrtj_*&3$vJEoo{gSA5*xN5q6`Le$inaQdw-o$Wx9d!%7KROx)cynkkrR@ z47NIaO07^@WUevCXCkxQL?=?Wi&S{2`;?w)Kb*pJmm}=A#}?P3RZZ<1nQNR*7C(&5 zsVOAbpf)}&!OwWitCbZsh-2$L9jHp&W^?dhKp?;&V=MRSJ_Jn&40;L$7$3eG$XGg@ zc7V0(GJ|KRn*)tfOcnmWlixUFV#oV@5Z1qu1KG=aJ#r=T2GCpqLcWF}WAoLSpIFe?c*fM;YBmo{Xx_ zOE8t9zk&F-eydw=lr~Vdp5PF6L!eD|k3q&qk@9JC5w_Bc8?2I7uHLRF84S7KWcT$5 z4C&a3j=Gw(A?@+;I4NV;k@6*l-Ec2Sg6D!C&DJZo?@D_OhC!W^@vON>UMzhA=VF*T zUBcoNtPxkbh+eL|TSC3|``jNcPm8go0?`v=Pgx;I7UdXr+?Z)v{jmbd1;4pi9&RbzwBEp9wcGo_yo!J%iREFugcR(>lA{pJf`e9f zWbo$TgCj>rRFo1GxThOTc$AU4AWMx&nZlIrx*Gg(clIW?&0H-y>Bfm+gm*iNX{0e8 z{aO}xqM%6Niq<1-YSD95|3ZTX|!C)?BqzrTlv)kOPzK~^B z&LW0-`ndqVVInTlH9lib|MPGw1JE1jb`ZS*?7L^7 zi!MgYGJS9?&}t5uKuvcY%WxUwV)ns2!r1fOc^>#50m%Gz6Vxh`#Nd46&sF2{hB@Q( zR-g;z#$1Q6OW_X}pt^Mk>QjfjhO!TByBNckjJA9QL{MPtZ@V`WR4tOg;dp$Fj(#E1 zO>;)yU%KiIB@0EYf;+t+Kb>**pt;SZ3!<9t4i|j*KP(6heEvYTj2-R`H|N=k|7?>T nd+;Y-;ET;NGQ!9HWgiyoP$nh=QqE(-fO5S;9!ztA|N2(C@IQnAs`@GAs{?x zMMnjG;t|ryih$sUpd>5x%-v+CVf!2L*wpFe4a-!**;CiCIOR;da#orT&mK*^*e>C< z-YQp_O?;nth;JEL$orC?IwvP*i7Yk{Ga!eS8lf4f=%XgS6oH1(#~kKxT1@x7qy;^r z5l>tX>T!$y_N`O+^LNElxe77Vg_+xp@J7$R+vk*$Iwdp7Ok|dkJRKm*J|Au3;=?9b zP;4G+hSmL#B0rHlA0j8Q0zp| zINiSrZ4^pym5_VB?h@|mE&a;5SLf^Mbi}b7QJCrTI6tbv)u>0p?Iu@$@7i*v>h&jw zR}>3gY!YC0Snb|?L!DF9cl!zJny9?6|5Kh zH1OTF30t(t+9V^rn%HjS)q#&p%EILhUbrc>mmasK3TcUVjBgS1A+eGTZ*3~bxonCOmK7)$-B(EM~%+d8{8u=F_8RTY- z>OeEDCYo0D2}Rv@U-k%U1z*_EXYzTYPQ2Roa$7y(FDV#@_ z6i*B+n4f^bJI_|PTw>SWcljd!S{P3}eV$2MtY3V+-o~_z{d!kdZ2cEU#8z&h%Sv$?_#(hw*7Yg`F1Tmh$;B65&G>NWx`O4`;U?Sgbtp`rx z|HEqYTYJ#^IBgJ9c1DcYjqH}`pZ7>)=xOng`b&;3d}ZPK%TYBb-DlX_N%U?{OS6wG z`x0FBwpm^Zln@uE7Nx?x0Zk|!>oeS|HE8Rpa8eWsubYGdJ2~T+xwVb!yEPQr_l{WmONpK@7RbboyarT?(O&el` z+r#RDO4oo${C4IIHF)o*b{xUE;RD`P`cOG#iwpOaf>~h!j=b#3B%Kl!h}+(5N=dPc zEaj@ktzJ{?L#s4FM+c*`|Wv3+T=@;Wg@vf?%TURC^p$mOI;eaiFH*YpVoaFts4fO zh2CukMO1wdwfx)IRtuUQ&)ct4QC{x9@cHsg-G?5r;Aj@s+hktHbX*#*L`fGLN`>h4 z{W$M8G@0O)>JYV2Zqs7s^B{>}TaoiwW69nof1)S1TdUg$CHeXaNvXs~DwxSd}O`PiXGy)p)pOI8tr|CFn0a;(7A!j$9~Y7LObQf5Y87#BZ;TDrt1Xy{_KNsX~Z*x}SUqh|E8mN($wbZusPzNuB-c z@cM@<5zt9)7xqnE z;+mi1iS$P9aB0)wTD&e+0rwNbDlh0T{wsGTrHQ(?e`k8m{Y*E2N_l0yoWDYeI_8xM zQ4IUSw3bGafe%LMzW;^ZNOpKSMfd!ivL6vP#dx-t>~z z|KtmCK@T$%VejZm*}RLK4|_1Sg4TI%o$O0a_Uh6oE)dkM zzKT*@YCikB#f0C_DGI4^JajC^SLH``%H=MsW#rZo6-H2SkG4i@j9WmRRjD17{XF8%KqjhWtA$Wa6n zDMx>tA#6<7lvsPO`iowjtFcc}^QhfOL?^Xx4$E#U-j9N#Du=o0cyV&~^$a)dxvJMQ zi8La#ltqmmXLG;W=SKEf9I7`HmyQM`xSlw8eX9nR`S#OYnst2z0j?6g*lF{-P3g$k zir+5EbyON^&b`m2K;`_(X>IDZE6%XO^K5U)bKzEs$9^2?BD;;$)GIdnb}vkVKVRbb zU^c$dvNPJ^Xzf=!5ZXj!h`K#l71jU)o5iyq{{A>qvE$%xR(9nt$9f@e)x*36w75yo zu8-F@YYWriXZ-zP(x*u~cJ>L?@2-NDLM3jLGGE=_OvXoDgWLt+X0DG~BEk=U%w;m+ zy*I()oh&oAJiJNdP_-j9aZ!@V6T4}P`xF`F*fGvL5sXm8g?_*97^K6E|yBO7tSkn zx|(E4!qrq!t5LQ}OBPwv$Y$h*zLJLu2`kZeejg#N)-A2lFb(&ITXMw0Gf+^D|y)Cfq!9<$q?T|(LQ#S2u zlnc9F!3D3lk`rLL`Vh*g^|;HH>4JqwGe8LOW>}|Q33$L3aLRV36aAXEW7I=6Mdus2 zG31+v=UciHd2ca_mqmoP*ret*e?X2%^QwwCV>t}P6{V9z);q`0g6Qb$NjQ8yZ`L|u zV?P8I7Zq>G23sQh&t2=@Ju~MoK4UE-_L>@%7pgDV_aJSLH|TuTuY01dcHJuAUsXI8 z7YYM~#z;7vV&dBL{RtV5j0^6ci89nqbTLo+!=l*3w^FBY_(^wi)GA@iV+J8otDrF| zrcQFuT+G0 z<1(9-1{++^a203w+1fnT&q)`zj&AoNHGQiO7X?DxsO|SWx;WAA9{TP#-1;b|Iqml` z;*)*RqrPU!Mr$Q={S5YOCoOid7g8Awx#wC|z0 zR!!lc49THw0MBW}%czke}3@Gi%;_i}4=q+juJG`5yjXM+fiT13>jp8_$3 zSuGaJltC3NbHqVz*HNELjFOxp%wy|`6sR%8x=_fZL$9ZtsZ}g65!HnAfG8cDPua@A zw3h+#Rkmy<1e=V;HC0A=ln)>w@TBwnxZeo++)_;JqI|T0xZyPSR=g7;RBMpNH)$Ko6sum@q*^nO_Ak?^B!HKK9s5aB;(j&*%aC|emiC5hp-3(*e^Bu2JBICFb#cNifzG2e!jHh z-2c=9@vT${DNbOP2wl-bPEr&-&MwBPuCxgnF2dg@+UIl^8*V`wjHkpc5%tA7!m@gA zjploGHjn+0zyGFc;79dbxZGy!diHp}D6HdS0_|M}wJCIjde~{xLBkyPM!i01lq=*> z_+n%YO3K}i-Pn7qY=4|KjahL!Z5aj=HJc+unF_=c`_MyM$;At#Wbx6Vk1%=P7hN@c z2WD}9LgX7}BCAn<{x*f#1DLHpbP==;!#yl{j?5Jmq{Z2FAYwkqU|Y0}c8ex`2 zQ2;hG5*>x6?+-#N51YtCTBBH>F;Q!x$GH7$Itf-zxLb_PtRZ}Bf3U^{=LlQzkmF01 zF7EnJ_;QCPbW@Bo($XKXPgIGJi_i)*;oiqROn~AsfR#fVcOFh3^G0-dDduUfDCv>Y zYaRM$({Cc3F)Gf}%-LlFnoBBjXlH8db>V$@^jP2*c^SS^&Rz8TZWt$2`!aHRnwkM0 z4#Y`R3Eaar*g6nkjY~o&k#xR(=i1k5qSs%ak+x6-Pq%vN4zfwf3q86tAR}6IgQwUO zRx)?N-l&kQe#EJa<3$e0Y}cai`wq5pDzbFA$1%zKybZ(T*9AHOJLg2|oH`BQdK#AK z0BlRrMq}VpekK^aL0{`VfcJij0)rPGARgB8bl1wV+cTS(oFKV3kwYD;w#@MlbmH^! z$QOXC0{DW#87+yEh7&Y8gcLXtQO4ge5Qs?a>-?3ji#mBkF>bNMEejtBG~*`}(NP1{ z2oK?FVdMN2%|pFV9=@;ZAL(3gA3%(&G;--o$N>BKNtGu^i|#!kllut1bx%wgeLN>7 zFW8ijiO}dP?)&QJ3x}lHId7D}uLHHAP zD}upW%t(ep=B;ROHTPs%QX~gW{qtHNGFe37phJ|eh)wB4EJsMRmIBzKe7=%{)JlfR z*=M~!pzkiCUwD51w{alrOEe$+^38KA!WVy^{dINKxmx~8%G5tLMzDZXb{~I&LRpas zazrB+P0J~yW@{d_eULi{sU187O;3C(^#enM%#fuf_s(wu4&*86%aBc2wxR?zSKdkH z$qgm8${=hx+=GJCELK~bGFl46#*;Fy%hpZ?RsHIDTPyzK=S+zlzC2?D>eNkk1!tWb zreGWEzIsO;k6Zkh=cTe$_i?(N>hsDFlqqycX5*{f5+uazgbUW7+}NQ(-MOEbu|W>> z6?g1_1zeWkhJ{C;oyiU+Bz<%LKY6ZmSq)nM{K*|4ZPCv;0_=adQ6$gz|GD(PX*6DO zIj{fX$~3AJQg`UqHT8ae2PG_ar{7&4oGA!8zv>>giNGLYTJJ`S1OiZlS%H&1 zFyvX8^F+y#zc3#0#DcqX`Qfi0u-5!z4;2NzxjsQ)W*78FJ}w@LHbAe4qFCsQ3+dDB zq%=&2oJzn}5_bSfdOm;VtI?NwBI&~n&=}La;*cluP2rp6$$X=S>*e9l7@5qqIB` z45idb_nb$$Pf5n29UCtS+%FQwx-EzQacADgLkhqOTRISj2fE8w+N-x6`{JIG^&9=f zQl?qnB**ml&GU?4ew!zlt;5Lp>Y!7iI}T6iA4BpL?-b!pvuK z$C0fzxQx#A4uPyZEjj_CLS6hKPp|5#-D0_;+4XRRQj_Fjip$5?$a(IB1&{s%?*VeL z-l3MZc>Rl~iR>Jimjg%bCB}7W(FF1#VKY_Uo*umF$rN<)KTp>BeJk}2hkKs+O8HC& zxGx$2WSkwcL2uo`y2~hGy>$IE7x5~;7_{@B(+cMIk(AM;KJQdn&)SLEhTpv%)qSvh zy=jxkNJk;Y8sB~*j2o0&u1;lY+9VD!>)AqZ*IoZ*3WYyRY^t_UZ>mocPdY*KI%p1M zd6)qIvQQ`GjLp-WJQNLfiFTFait8YcAv^MKWUYI=J~ROoF?NZtS-B$AW`M zhhB!qG?@}qvjZ+TZ!2O->qAP!rYWsulA83K*f0~ENX%jyFvXZWT;033e?6|u-`J|sE661 zR3=y3OHV1R>HUP(KNV)NbQlR&ecCw>%5ald;(3Ct)GQD!vA9wIr@HQ39AXF#8DNZNLo`Ke%Sfl;%0%`N0zsY;1T3c)`JbN-*>3<)>;+;{RHjh zS+*UK?)R*nH4+n+mrBmJ>2dC&1hHI#^8Nf?+|C|xz50xli7nN{4Az)wPcllpQOkU5 z(-U`wX5=wNth%moJBu}qxdvK%S1FJPWILA>kdnr=T7I>Fo5IX3rYF&S67A<0L}@*i zBYH*HC0`orE)oyhlfR&@n?7v$XO2`EBWa#>gVM%x$r{>ptXWs%_pv7?!?#)P)9s^< zrN=yzd(_OL9oNyIr6a6G6U6C(Yt)NmSkeun>TdX%uro2&>w1#q?Bxr+Oy|7|7O|Ey zgfn|&Ck3A%r@BjQ{MUJh*AI_=Q-ODnzoicIyV2dM-Jz#O`}^0-0F(DMPz+e5&V4bM zyd%qR!rj8{$CJmM=q}BHq)ex!2|aY7<_U2sV48ayofXZR>4|6si!=7<= zaMNR2YEsjZIX=zu<|GR_8%WJK{lrI&J?v0_F&yk*;*G zq6$q|Wbs-XEr?xbx=_Nn1;AvWYEU^kI~;?j1LCeExgVm$KjKqDBxkt8R*zk&8d5Q> zKt2!E3BV?3xuCS6RGewNUXEWMKO`Ey*%8Jg$@0B{5DrDs*%c#C0mTr3esi_ot9ixg z6F7Z5{Wd1_zdy|YL15f)rI1Pr?Xu94oQc9Iv3!?TF3kpBJ!|}Xe5M%Lz#dxmgAsKJ z)lGLtrPkcX_#t_A1jK8!&Y)^Mp~nP$SLU2_U**I}{Tj2yBZ-`)uA`H1d!$L~YS1<$ zVBAS$qFnM#^CgM>9r&Y|=M{yHJ%!)ElB zHxDo>l;ZZ8eSAlGx~uP#K(cvaGINc`U5`~$KMY%`1WI>ST9Ch!Oh+GEO*C}0&m*W+ zB+oXcl_Ei5kPf{|06SiitrRxNIRrITzO`nRq4RrS3Hw|=Q7~}USI@N?+r`XqN(|xV z=Ml1oDi~n!ZqZwd>g<)$as2!XhE=8t0MExVlpKePi++^(BSyr60u^}VW zDWY7vNf;Fld;WKUGJ&O56?53UcAfK9Wr54yraO%rp%vjXbckeMMT~~MT>XH{O=Vrv zHMO=VBPC*~RceH%Sn7SA)42U-=fJ9HyGy45z3O|d$!#xT*n)I=Ch z5=Pw$pEDuj19vW*4GI!{%_;X68@L?=~rldow(&r)c&*QhrVXf6BrOUKj+ zZ3wd=#_4#!}y7p_kUOL0{)vhxUyVPKfbO zJ3Lh}Ct8HmvVCO$ZZANPOK_r}LW^X~R~2#7j-@0-KJk+;*)c%4BT*(Lj0DJ^Ym&nQ zB2gHMNKN=#Y(5cTSwY8@pqm*y(~{Nn*2-qxy@{TTEOCRXZ77|&3LIHjK@?h#H-kim z)zQKI_Do&1HE7cqyd;S42eXQi$@1iMPUF=#KDOc0h0i8=$%n0-{4NE%&!O`MgZb3e zB#w3-o6m2Gf4gTfAqw~JU_Pyk*UFcPcl6|39=x1LG~A7PLNZHo0oXDmF)8&#Lw->K zDobqP$Yh0wo%yR75f{}@zHGB_2$qG?cP(N1fWCtQSlP(G5ON0op>|R+lY+_s%|I{@c8ji6didZ%Seq ztezduyuwx}`0B~t_GQ3oisnU^rI8ve%oDWUF?kH$qeM~SBDLhLD^QEYlKH7kySj)0OiC|>GXs$bP-~nb(kZuVc3y;7~UKI zLwc2Aiy))#5e8)JT~UAv8{N{JRM~H9tE^%Qse(ou%VvD3h8~^YR^s3I!7bt7L$@VZ z&Bp}ZLsTAzN?kf%Lai|R>0Ylw31ac7O=@6<{)EJLUK)(*JJ+i~5iO-nVF-wLX>DIg zJ`SmCbBER_q)4pT;G)+zub69+Cq$q{@pmYcDZYn`V4ef7<~>^C1OG>^zrNlr^I-WA z=NmuW5a^R&Zga1W1xkDLiUw5ijUTaASLPZO@gio`3V1<4u1}{9%HU<3;vUaQ^1nX! zNkC3XWJA*tVpQE9aH5)~-4C38*!qB&HGe^~70a)@=n9k%`mG6Fy!cQS`Sgo%cDaC+xL*y$5B4^pTCxJ=RHb)<`(H?wkkVpb!AaqK|`wFjiRqSqm2hg^AF?s z!jnMTdx?Zk9QkF-e}bRdF%;WMHmP4By~%~w% zAG7qX2yI{LGuQbSVzGqKi-B2!Q8{bHB0GVO4eJRDslx`X4VNr$ik7+}C$$vXxax7` zoC~*9Aoo5p&iz~YI3a3NyOH@OBZpd&m^83ReyEU+@_;TRXBUsq>gP!a;jq%YD>ZGT zPW19_*c+<(l=yf{3T{vboWz^zgfO|(Sfo~WH4`(7;B4`$qLqObUqtqmd<97m8 zPckONbZK9AvswDexEg&TmQ<+l73V>U#ymKWtEId`_{pp%LaCiTq}VK2rC4neb9$}> z=U54xx!#VY(A4p!Q?c;!9*4-}Vj3W^cgZ8vb9ev5nGcG6j9D?%r+7jj zd}(>C$%Sl6Z03Xuc5x5R+30R?d2#!*HTZXXZwS+Um6So|FQ!2Mizz&c=NZUVBdtsc z=H>O0ed?3Xg3*6mar!YulnMESvT?E^9k*)eFbnL`U40OiFIsw2pL|z|B%}U6Om+JI zrNZTzBI@XKxRTQjdHb{xzV*eThsurEexjroPas+3wa4M)vO0gh_xa+u%lvq;VcM_) z(w$268DJJOFd4KUhs&4i{Tb~*afEfe*pow|()74iqBh0yX`;BHO0tl!y0}?=1$=A? zV8`p-3_XfxN`lLyTqZu&?^2zYfb=xHu5f|q?PwO#)GhGLx7uxo$#;u_X=kbre}V0~ zDBty!x2M?LG5|njZ>}z4-}@G={i{CSN+_YJ*P)!qO&!C{VEoNstFtydno2bMSDGLx z773?HfCc}-6;`j-93V+qDHt5}eaLVB{G4mOTa51XxoEZcopxmDFED?a;aYwd;cJB0$VU~z(XG(M)t1OYmMjvmObmAD|K}mmF z6YU8LyRTDwrVoV6n{ylQ{C6OpHZ$vqZj)Q!8fad-Bhqzc5n9RSK`Aiq9-54I{CrA4 zIHu90^8Azhb#M>uahyE>Se^vCl}sH8hVlSD7_Ik<{t+Lz(V*!u)5*X#?(SOg8to=! zXI+l~=u2tSO)bkl02LJ9`PZxaiGO6@lAgcvnp|+o;TVlO5bo(&G$98~LbndaXB^i< zN1WiP7wtyy^JQ-_uVWPNZtp&j8$*|f@sE)Pxw;MKt#-=6@ASt^XDtP)`~3OZBe8oYCXqlUnbE{5)5 zuNZ*P1hMOkpyn$7QJTg7Z%ecCr286eKxx+HUrV$3Zt6?IOxLTU^4_=J7s-ro)NZyH zQejZew}=4*?f(E2Z9E!BBK@gilcbu^(6Ac?1+sZ3+D@Mb%zi%qke6ua!+_M>!2&pI)qI6<%^t7xok0_|bO9 z98tBc>jZoqc_%t&h=fF;WxC~Y6}4bb(AadI`JVQHFq+iD-X$WyF+QnIqjQMgV+@x8 zsT<=v*#SH0esOpzRj^yZRx)cziA^4t5hjlykJEvTiXo4t?qMCaA(qeFbr4C{!R>Xt z0(vXeL1fV%+pI-W5tc18-JWEU+0zb^mo*1>BQY}A-W8H%w*D591Eu~Hk`Jg6I=C-E zj$tg*?yagn^6u)$2+auV20Pl1Gx4H3(>e3;@>w^@XACIVOi_9Ai9~5B((b*Zu1`BslqV6F@ zD2AB(FcDK{b+jQZ6ac2<7_w76;Pgl$&`KrBXry86+VBK$N5}~jIa`6|z&kq><>j}#065v?&%(;lHR*G_ zfJQu0Fwmtw6>5u&AJAF@ZjLrnG4u07o&t=(Jf>?9%f(B-8rtPg4UuF(^d0wDRmtw~ z{dWL%z(DVy=4k(@ce}(^wyTR}oLXZmG=~jlvrhw$iY4&s_y^!ZJY_@6f-9&an8h0l zoK7EKWTu?ug9*v3+ZyTdP$@#vA}PP)NRM87zM}(qT0^GZy?fD+a~~f6OF2FYLApwa)KZw&yQ3k>GN!@LRi zn|W)=mKjdUtsxu}mEH3e%i1wi6KcFAJt#zCH_*`^Fm6OP7&Is5^9y!u< z3UE!o=Ue;ILUMHZdE_i`4Sz{v!yjjV{Jjyo;hh1sNqoO$R0Zrbor2JIGgsDR8Gbj1 zQrq!@+uD@kv_;fE=hDXUq&EE^K}ngf`Ou7Y=f)c1-w+8#l5^mHpqQM)fhhdKys+024JGjYbUufH6m4y2db^d~Cw6@F-Mna(h0mE9p+ZFTk>c zVaXqSgQE#WoRE!_AGNU#z;^jH$q6UrVl+24Leh){hLNOo9@%<{0AO8-5ubn+@S!GG zy>!rkR^eSWz%BzhWu7b-Cx}g}06DS#RgJ<2UJQ&j#r10N%elT8oq)(1NvaeZc3vsr z!?SCPEM~^ixl6N*4h+3xdK#wdE5JGV*(*~snsAvgfIeDZe6{Mc;_naR=F+E>mBv7) zS#`dl3PjVI@Mi|*DQ9Kf4_Xe^Z%h(&p@fj9e~VhPm{`e2+b*|}eNYYW-ErBDH0O?0CtFVlo7dC4JR@3($*f)yXK>>KrUe%A-2 zwir&`;hz44lXat>3C2OBr=w zjXAzN+{^nKpZ2*QgfethZUjnqpvScSW-Lyi&CLj^XxHFg)1OjywH|D&H@VxK$oCOj zG*D7Vj>EijE7bz9@patSi_&E2pjk)AOHn*wuP0jgD;v@vGs;lJ{emHf$V>1J*D1zP z$To8%ZZ+<{4E@`Wq|5PG0nA-4P37FmL2lD7zK77qc%Cblf&1uC!GyB~WM^pAwg^6o zzTm`bH^;*io=Ee`Absr@S5weQM|dWK%7-S*i`})$=6j^){|~W@%6F-|{c}=p*0C!+ zIUUP`gBL;K zM>6`etKEI{0kr`)0JGXqJ9*yfZ{i|;5x4tMq#!CKp8}(P!*TQ38$fX;xTj=X@W4jv zNV3xESvR(4;yKhq-9{gvy2hcbU6q9N^SH=pnfRwuwFFjMo=-UV@J7G9LQz}eGhvs$ z?aEzvzx`sqipa{~c^pThGjh3lvZ+C3P$*sGduf)F5(BOd7E)wQsJ~A|K+Ug)-g*m_ zhg(I&qKBXnU3S-&Mr5+xL9gGUA#T4be?F~OYb;VI+;*+~QT_M=V%8EMLAY=525o@s zvX3ph_M4p4%;vuju{>0a_iTpgS1Td|?LRopv6>`)BxyDiZilv8B4_ak|Ao>p zkmPDuvPvbMq0#1mbjOfN6~ObjWU1)b88Jc;*(E^~YXR)h03*)J~y~oo3Y2SCQHfCv(}()tp4Rm#|8pN9^pc0 zu-XFNLI^e2%XIC>U5w=4AH3ARz4NPEv;T$ROySkyqL5V^n23kkudodu{$V&nqRoF9 zP9{pEFozZXqDZ~xhe9l>0h!+E6Jlxoj`bTq_ZNsL$nSr~wR~U3qx&KK)D@2u{zcak z%iTJZw?Nt?@?2EIqYU4dl(XgC3JPa#(oq$a9reC(N+~fAvd&5Rwh2?3H^TgKsE&0T zs}%$*>l~&Uh|T2&b-a_Zau>qwPcroj_tEZTSV>KkVnh-v1_YC2tKX=OEs7(YMz8pJ zw7i4eqPBlWEen<%-+;VFv6^*J4p;(VtUBpN#?8jPT$yIr~4F zF7UB074AOvUStC_1=s(%^#6D?rcBoQ(@eoNwoaL|A4(hHpOfEq8n;h>--gNH6sV<{ zfKShfRPAVdss%EB08MEH$3s6w_v_QP0CrP#HHer!(>Pt@Y{Rn`k`@lM!u8_qk~Zwc z07&FR{s0NlW3_%jmMHz|^S#=BYYqYn< z`o7)9GX1bR+nZ}YYrGA#82^d}lX=h8cur4sJ4fJatL(e6Si|Do8BNu`_A4y{1&j2t z6c4%kO|bPR2Vg6!U?_&X=pMEh`ic{PC{EMOezL`rH zJRDgizKN32C;{9L=-cg|;xXW8bo|xJ9U@m11yRRBa3~_qe!Q|%t~tPW1; znLJ-JRE6L^Y$c#=5?(1=nmyv|iu07-`Qqwz^h>b)uIbQqZ#scyn(MFH^1l{H6yk@* znHoN(?c)J2UB?d1eFXL-0DE~&7mdcCFXPPCr<$!Na+Ry;ikA$IMJldSf4R3-U}=py z#+2oy#cp_GQSP*CR%c@T?eY7Ht^uFJ?WMa`AqakKyg!p|dw5bRY*E0l!piP&);E*O zb$5EsC6b%>KNNK7T6F&hC4xXRB=EaB1#m1teQ3j)@Nj9i?3N*AA)>8}yNZ9$wDoO3 z{ORWZ+7=-j6{a-3-b#J87r~~Jx4lwSicA-&F7b^KW9oz8@jBopgG_!eE_@4URP z;VZ%HUyyzqX9)UAZE5S>EQO0*!%W~t<1`1&Xfy>H&$dc^_ECTF)E7pQ?RK@&+35mx zx37n|JTx0$D(v4^6wbnfvG zY0fV0TA%afEdB(~o=?7xAlDkHU~{*&SOwB1@HsR@$_wZlCB3aZAC^x+r|pMqZQ1}P zaP(`|G<%GBf$+{re6)Y3D_Z|8UD53R2VIfm(fhA-W%?g<=`Mq!NB?Utw{+oHa~!^zv;3J5zhPXI-K-<6WGE-l>Tm>VHG*5!WeV-8}-l zzz!>>99kJ}v!8TD_>Vx=V6sA$hQfz9tYFJ99_(G9CBim71&g;{E1h9LTEB}v6YF2R zk7W(O`+(egAHo00`v^Hh;0^&Gm4eDp2ZclJqkK)SE)?{?-beEMAMfK1nMB$|I{a1d z!Pp<&TCIZq>3z^tss`xoGT;ozvq^=ZFM37u;pW}-2lqH{pJUkho~ENk3`pF9-`jRT zD|G^1t$BYVU*~2zcss1qzm2*ETIb^;I2gqll1e3U>uL|}AjgZSkKSR6gkH!?4AQtG zvk=j#QDdMc9R5rX4ZrJk%U^DIgcFl03HneV_bfTE&-}Oe?_^o-qtKe5vJi(mYT1&^ zza_O0#}p#>*};8~_VoRp>e`&4V^hX7J6#vO2=}N{PUcG{4X!PhJELBFeW1U_155d4UfEv_YMP^Q^oop z?QjKjqAZhGX&hQ*006}0F97l6zv=;~h?2IJ0d~38MzYREE-uJd`d!WH_zbyS5a)-g z^G}5%jggDYAOewixjLY-Y7Nc0A*rGDv|H_Y79x#+f?AF}>-l0n38rWjPjIP>%KPAH zhj$*3^KeQBAe!T#%VQt@lYL+xp-(X4$qOw#SS0)b67QmvQE7X^02T7>$PV8cK0l! z;Ljd3{V%o$ErV5rGfyJJ-{yjRoIrlS_5f7}u>y@rcfJRbz<>2Ua^93vM#^G41M^e; zk^nM_5vXTwq3O38pz=rd$d;63c@^b#Tvty=zyC!aug~sSV7&rcB!mYbLn64Kjyz4= z5Q(fqR=Lp_Aje-XTivxVta=4${Wp}cs2QQ*2v_}!GLEUW#ZB9&+*F@lB=-g#9i|9l zpr18g%h_c$n#U3+sCKb?vp9|as<*~{St553{{fGn^o<-`s!ja;nHkw!q&DvBj}n07 zF%jppf{3_STh0z{(0H=IsKMYx`X}&X_~S4X(_WT}p(aWv*V18DpwW0((I8G;8W&p7{50_8F zuCw*(`P2F^p>5z`h9?JYAKSv`1EsODVd7(eq(_md`G)ldB?leuxk5WRJz#XSrr#SK zCOH3Mbc8B#|AodvQSSrPJN=PBt0l9rxB}G{v)->3;r!}{Co3+j4Cw(wl&GX zpujP%mV-w58-&aFKs1~jTFd8kR}C~A@joMOJeWe5ZqY@tmFuz4aj{ikn?I7@-rF0k ziE{Yv!!c-){0(ezSopBF9EoCuc#OHjUJdWfKXT%nP%kK30CY1k*3i_P>qJpR2zYVw#(HwJF_PKAaBTQ$*2u{M( z);Fl45B}Q(vnY2_98t(2jgg zebpDCxI?s-M8dIXj&7Nwq1^Cg{Fl%tyw!}vq-el_4c=-ONb!tJX(74q)=M7xgQB8n zJ+1bxzz{_<_D&SsgC6|hg0li{#@xeE?xIujbH6S+)qQdE^gA&lOzvLH*he=1smxw( z&fLLtywiF1g|VyBNVu$p&d$SEt)mQ#@KPHa>zQJXj{;69Vn`W`;VbvgT-w&7N9GL8 za>=AiI{)x8D(&vQj6@EuZ8l_o>=$EHNMw4#vHWcq%(I-F6(6uF;;Ra&kMSN5@D%>h zGPt|Q?h<}GAmM-SgBh?W%XC5?dAUFKUCK}H}Y)~&l z|M1w1?uOGSXuAu`NSRt4Ej2S-Fbmj7IOP~t>Y?{tNlI8T!s<3V@+8NuWc*d+|g{PAtfvpG8C zxHtM85;r87ck$EbfAoYXYX9j8(fp@u!K!$k?{n^U_4-HQo8J8g?gAJRVt1t~k_1G! zIWt?l4wW=M29sRm6wq#xWe7I8za3fv*E}-yK6%cw8eJC-i1VwV z+I0=f@3OC*N4FQBAg`b zMtVCycM8jZ|I!6%Vty^KKXgS`uB8CN!1FIA{rAbU$5z|-`M>zJK!ng5V-eCAZI@gysy?$NQKZ{de;ebZ0Xcu*3DCadc(Sz+ACJMNGJqxt0I^-!~VOBz4#~F zzvSz0p4{i_BXJ(swq$)qj(b-M5yV`@Dkr2Xg&YfBN2;~sxmeUsscRKe0K&ggsDkZ` z?ed%JwVV(o&@i;={4i_OX0_+Jr|f#p5}=2?%gexI)Q}b59f+~N!jO@5=Prk&Nq!>s{ScT`H}~%tCHkKxOO%wV-jeFKF^GJ&qd-9j-y;bX04+iOt=f(~ zlt9-Ti~2(=@Q{3@`5QV)SpdpV3M)jsh!1A((U5ti!6`2+&)k6@0_xH*gzq?aBh>ni z?^LR1K;Y_sjELiB)C?)!k+P9y(_Jk%dEFm>p^(k9l(dUmE8CWB1a`?FKty7iBb~y3 z=49aurWa}mHBdYrQn{>Ad-xh+$|HX4-?DzrhNKzr)%2u&(3(9Mdr}YA+ZQ-?Y{X{m z0px&RW4?HqDeuaij@$3k+1P~9gwPpqicR6N0kVI2%!EvlQ~F`HF{n4bxgKG<(127B8{zXqZERKI6$I@8OkareOY^ zZ**mcxI;g77Tj)kTyUim;IJg;BAQDuGC}HDpWzzU5hxovQA0d8+K}b}jLn_gqI#t5 z4Y`gIEL?`)40<&J-Q#vy5Qcr0oSQQRsyK1C)ZR$%bcU5hxNcNKRV=FerZ8SiQMKFJ zH4BcTwcsvQ|5?<1xpPSJp2hsbVbayw>-CfdMf*{f*ARyf(T?<$UNOy~ZC?MS|Iu{6% z(y-`mI1~5Y@BYV$59d1H&iul4F^M(C9CM6k{O;$u$GZ75caFNp0{;WLD}!3zXI)8`xaouT9f$Yv|L9Va_5}LTNLEg1&An{#Go%A~{^?3GB0d5~M-E07Okuds6^)sQ4yB+f8`BI4 zG`~&z0f#$Wd&v1TK>cpDWNhLc%tXN?!3b>kG*_%t!@lV0Me7E)`Pa2XZi^x2NdD%d zb^%7Utk2L3+7W9=id${oIJ)0~F#<457&WQ5iF~Bwj?b|+Zy!E7NfuK;>~Pid4a8 z6|Hz-Lv*m`Cf%gKevT<2w;Su(6KUnB^Z^Sx>M{v5L(pop*+yD7QD^L(UIoQ74Pnkv z#|n3dDK&+Z><0&vuZDS;J_I5rMwybKwl6axB&O4}2vDk0$M@Jv!X3aszO zfZcDM>AE>txwEJ);VR8oC1@xaIdi&C>=w9NyE-AxK$*be2Feuzm!(eELyb0WBKgBP zKAtszh2q_Cmi#xGkQ?m|oLim#lqQ3a{>ns0{1vpv!|s}(B3AJ92&fAw;yn$FO%t|r zRp~v94^ex@8sx-*#_OYOg(3|spIAla?|b8zR$kLs2_t=ICPGWV1N2TRfDo`qB9SRb zcueVB+7fKSDIRqE#vFL$1(TuoGJUWXio1QiMSWizg#&-omLz)VpQG2jQYdt*LJc_8 z?Pq0DCkHjqQ5Cze^cTpLU1EYLWZi_*V4A``5UmBm0ZK2w*V>hq_a_VwnZ?(^N7S3} zIBy1he6_URe8qkVerjEIC*$TAgg}}@J78w54oPWLu0g+6yJP3SV5m7HS>6L>?ytUs z{pmHxU4`4RNcU0%lY3`3{lvD-h$05=Pvs@<2PK`wcN^cD`?6$B!t_Z2kA(xcPQv~Q zuYv#FYk(6S(y~_2yf84yu22#?wxcY!P(%}>@?ubkzajA>f$$aqI3&7?`Pizgmpb*M z7w5+6Y*-hIBhYCG20$LeNvKQ7Kc)(xC%C@fM&SrUJnebVw3gP}pkYkPAs@JT!tI_$ zumtpwJU1{M->64HSk4C$;_g0k{f=%=+wkWI3;ta?GO23e^6T4rdGpqhP6h%g&66~) zW-kRX`ef4Bt^NWhl0p(!BoyUZ!bF)Cf@2gi)y11&-b8*10I!;vR)jpeeXN8@u9FG@ zMUuTHkaupAuc{OmcCnTZ&w3UpI7~$!0oFsYFS&jM=9W>8ziEBUJAP)b9%hPJZr++3+zElkNt?e$Ufm+4V2LA4V=x%mQD$Auws|omoN4U8wOAUK09K@TLI&w*=M-e&dxac#o$!45Z&O8X^tqfRFlaOM{A9Oad2v7&Dk;i6ftY zesg5ZPpTx&Li~F7`N3$v(od*6kd&as%C3SWDudjmb;ToyY7?VCn?q090F*NAnS&m4 z5<1wpa5QhMQQj6h(Nn!XBRtTj6-2p{QQIC*=OgPw4FV%=Ufq zZwpQwL6)OoNHa9#S;#F#Y};A!c{idL(3Q{o=z0rL6mP_P4Jr>NdU8eOEg! zhIoAM)WQ3muZ>1IobFj|1uspP)nlpH+M8rd?zX;e=?6r@S)1)b>g1IapUaO%AOze> z)KSd-{G|5Ws7|c4O-r$x7OAzJO{QfP+W?!GDr_u*x1cn_ukdB79!By09*p)dEcMKz zl!pv0dxLk;s$1Uye{Fi&g6E4~(r!u1BX5~13;Fp3{TuFYLoQi+*`}rB@WeQU*Jmk2 z)Mf(0fpf~5XM=hR+U>9u?}+4bi? zTc&YjRHL}gMuYHZZ2&c8qNxhyhl#}RKj7o}-^mHW2DcoaAI&gsEQjI=zFebK5t$o= z`QtW*=Plp0ns|R#AnWGIsTSu?yZ}dhS4+W&W7Q@Cv-QUS=83F(Q6L4b;JS z9xNLBAKXYWzZv$WkxB-b2-n2X66H~3$!R22PUeKy?u@p>Vspc*ug>PYM*lbd7-uno zKYyg&0-ks0>0*ic)1Q?CpgVOKXObkt{(ynq_PwtkwB%OHl+^L!EI6d0pF%!)3P?h_ z0p279;=hTF18f1>wD_7wSGFcIXb4~;^Sz-ie_Sy3{h z3E!qX$25cVcISW1ax?SEwLLmtXtDq6d~s)a4|pTTSPz=A3%;XpAXeZjU_btDdfQmP z(YT|N^O%z{(2idNSmmi-c{jFN^-rucJcyg_&ek?N%Wzbwf} z^^5aY2}EoruhU^%eVT^cAL{>563t^e>kyFVhg@Es`y0ke{%57>er;3Kk$lWa$|~f5ZjWuR@ty0yyR+$@V`6>& z^&ZRKt8w_wTN-to^ZxqZEIvS~<6hj!VEQ{4oTirO;`$ zJ?D+N0XJRjIPl%!*YR3^9LBn&){<5(ie-Ckox>H)8GNp?;df6*9;f?|h~60f4PlR8 zw~K!2*3T<=o~8`ePqdx74TX{v8eQ80!7);Ag{pa;TdL%Z(WfrJr(}s~H(pSy3UG>^ zb)yb2yjWhGKCOV5eqMl@)~Itw+D2CPO8RW+>@OrpQJvCz@p0}&xrHB~Tt9oD!r!}& z2^gTob|>Ro7`^i1aLc-)*shEF(uz$}mvpqYXptj(oioozm8F;|_v@k+SZBsn|6XT8 z=icSkB%s934c*wRm+~i)7m#2`-5zOX3qo^$8Xp_>;}AORO3Q=-KF zDUw&;Bsf3<3uvdX;Z)$R>5nvzk6PukhVX8PUmi+Yh6i)Z#-WxOT}x~{62Nrsdu80JjI$8_784f zi+>K*Se?KcPH`KNXe;+`T`I`ip41;Qo$$y7p4_ZlOFFUd^H667>=TAk9rC#V>i4th zqv|s3nt)?jNlm}bzS`#6UlWDU9|vO~>*zlB;o`ymn(5OX_QM19$>$p!Ol<^s2>Zj~ zRq05Q?p@*Z4bBlY9~BvhP)^?eIty^mXm^dr2>CX~h=DqRL!KO~9U9d+Gm-R$l+>3s z0C0G=%N4aWO7dJC{!vo%n0qrK?Oh!N7--YG$dEe$9;C?%Ei}Yd-lg?KH_0sdE*GbM z6E2US1(txTTt*8Q8~EWB#1es5@{B*@+%4PIc{8Brats*-2i9pyx{Jqf_lZj;vYLQ5 z%u?j^ZukdCGdCT5t;15<=SO@l_;_`=eXD3u-_*@`)hMUk%_!E6`~1ikMB2MH56WjP6q$wyamK-`$uy~q|@Kfsmo~Y7X%2$;8&C4Lt8i7hUmaJ7Cr=`nXD8#4$Lx$|^BZKu z(&=h+mU!lM#?(y#4Qmeq+NmD(!e1Q`&r?6Q;d#z>e8n|38v;NiFQLmHEMYvJXcb}v00 zjVUHKu~mb)>6?1>41R{<{(AWEul=tIv3Q%G|C573lK58+27&Im6nClA#0lM!CE^gmP)BS69YS`IX{8Z~cv5UTWAG=&P9?vOHnR#aPYNSbn>7Qrsa`{E}(Dlqfj=if<+)-vI z6^p!*g^C39{sPF6zSLz#VKg+2E3f)dn2fZQ@*p^!fWJi*&*B0odVrw{yd}*qlTKsF z$z$S@w54eEK`P>UKqtk|;@L5T#s6LXg}`2_mQ6FM&cq)J%dWu~VOzwIC?CXZNU^ch zmJy775_>-NpMJ;KueX1#(q*aUb9M1)YZAh{auGa|3y8iDmjGl)6hd=u(Kyv(0!KMf z%TKbq0=*Cn4e_h0ZR(P`lk=E34LMnNNDCjdK|DHME!V|~&iEm%R(@4}4KNFM4`$SU zQ%}3E)b%xn+%8Pz={oZeGIy9Tm4tMd`Ui*@=X+XCOO6m)kWw&EvGC`$n}*BAeS zoAoapUpsE{{ZGXzs1+Hx2&bdlp4?afg^=F4lnP=@;HKvSxYcHe8sg&|4S*2v=3TRd(^Aplkt8+=c zNcIEb`m|@*t{QTW2$q%MkLmQ@^C&WFd2J+Y`0h@7u1-`^^Xube;9f^?D{Pfiy+-}! zX^G^Yt1Y-4c2HMnOfD8com{Jf4S)D1_R{Q6;=?C-sTu{TvTPc|mg}jA;0_t;Fj)^- zemO!i0!-w8p$_Du%%L2C)$s?j&(XYH>IZyg?|U%|rMZ4$>w~LvBk#?2`@aq#rptJ6 z6O{jBTG`(5n+ba{ular)T0l6%Jwn)7va~R8qY_cO2*u@1rtiBpFx^0=)tZiLqK$s_ z+jlaTX&-oC%eIrjJBJ$7R4UzAe*>LGoN!ya{l2+^L){jkWb$5=nHf(>%dfqH+~-EQ zU{2E9sO%^rIxNAi#gBP_0>?WPLJLURLRkA4nDMh-gAv(V%MojnnPajj3-?Y>H9?Hi?wP8qVJ1 zNKQs2j0*Xj*-BPPLpP~!L0Dxfc8~3MK=xVDrPOC6bz$_zL(oEq(c+DTWkI2ow@s;7 zqtGmLA;%@5rLHvH(<}4ncRr6Jx#?HCuRJqxjvyXlcqGgHi<_5l(PAjHv^X*-%I^^x z3!1{hDrr#X-%Z0(BR3f|TVJMljP!pP!PQi_U1@%4@MCD;cJrZsXaM>Vk|j~5T8fw+ z8XDR)PIx^=77$jb0oc|z{U**I2el~=gwNnk@n?9CBgtW-+bA`cnZFUmr9*^qq|&KJ zA*N^5%#bD)QavPT$EkGpx;T||*)7Q-7>KtdBvMll8M6^txhCV#pk?57PqH9ReCc6E z84zoAjE1LEb`^KwT%B6o+GDEnMh}2sZK)b=8eUnzvML;Apm8Kg$_w7D5GqnkoLNA>3Ps$D;e&N;Usez;VZ`4n4dn`#7&gJ zmMRFf1dd$s6;@orS#GBtI;Bh^iYXgp6WPwg75FmAtUWP(f_I6ea*-L!GM;*Z%HG|6 z(M{v^XOU0`M7`!ciBWP4i`+R^qa zJkr-slg3|kC%3M{qkcR3X%Dv)@r|o;$tW`5=m*&5;dK6M!nhnl+cEGlmg^%fCWv5R zXtYFD#>a2PLqLAthPP-h+7(?B_nIv-tW|IHMkp8>pQTXRAQTczm3Zj5~c?8s# zD1S5e#vSIF$NPHXitIvvXbLk4YanE)2jEa6qcZFlz7J)rMi3@Eib9_|`e6$=hjg-i z6{WnXaS*?t#e|jXXRNR!XG|29s|w!;3y4ycrrBv%x0_GOz$uULjwJb>rU)v8@$<@R z)>5cQFK#{X3ak@i5fnt=q9TO1_IzO8-tj?Q1LJ*0% z7DBz|{~7UnwSDPBs#sLHsY~K-9S|^ zE<~paHCcJZ;-=V%0GA&P)O`}@!4XC=nu7MN1n6iSwXDJeG(rn$g}!w{HYQif=+b#H z+4e+J7!Hl?HRF0{UU}nDnWGMw+j$;mwnvB>r9x5Z$iA2`aH%bhmHuQVjejkz$NJlS zi@GvisBiq|@v+@+s-w5oX#Dp_lV#BesJw6&(3rQzaJ`U+AZASA{%(8G6u67!xp?=s zkd>>7v1ENvNe`VKLTkr|LW0QF9541@```S%cCqj`tK$eE2muRH%8X$b=N%7>_UNxG z#jv(*pIE=H5cNcD@XdRuBGQcJz*T9+uN>E-BEV7m?9GC!3WicWgu22N{%#O;7lsLh z7!)dcwH%>(Y=AIDi#cHJ8iM%C90l|OQSl};5ln|`AuX6Lt7Bq^ z3*Q>txO>coR0w|*LaD<5R^T|sdEm1miX8hPTP(}kr-GQ}9DqGtM}MP>f}){dtAtMX z0W@ISXm=|yql*&Imf8>24=5e)04{S_Wcj0CDVA(IYJA&+FkUc=;E@+gk^uiYVc}~v zofSW_5=vUz%SyXvbWDn@H1VXQs-=v)D`V$U$&7d;iROf&9JQvb+%GzH1xsSr6+k62 z0IO>?mpkMONP6%iN`O;OaTZ=2OdP4`N>o4wuE=uAlE*V3+?@Y&3Bx*dA5W;l`u9>a zPWyw&9zWw>*K!%Uj=VjEN67AvhuhI9r~B<6Qymu%<)Vj+e;u0x;z2xL#;yYav6>lh8xvqpYq4=0~_1&r*zf~rCc8;$O0k@WxzZa`~ms@+(HZt zDAxj9v?=4!q#?+`+k^an-BIH>+))jG@w)K9Sr_Hosr+Aus92UT7R#`F8Tb0nQ5b4u zDt~^;?4Ogl()p9oifa^>{?tG|{`bPH?53sipYMDtUe~>~JnXoMxN(-+&y-Y+e-XTy zhW78zrC}O+=?+die6HJ{(b<~|GGlJVuE)MuZJn2jb1Kwj2Hcmi{<6*Hflw6FMCes5 z>Fr5f-fPV-zrS&PsE+#fB0u@(me@ZPR&E5H?$Qcn@yR6k^SPT_q5;+a<8Oo?ViH4< zpm&OJIiZ`%7t`9T$v_t&|7C_xVi_;<>eZ@q0N=;Z&EGx~cOL(@&Ft*II+4|4Z8`756e>&MMLVW*8^n8%Xrr{u1 zF=FHk-Q~T&sEAztp9jR=_bdXfi+)vz{HOlG7e6=Kw)ENCcQtYNyMoyR95FmR2RY6;Qfa( z-l^T54gq_tmNrb;o2J9L;Y?YPt4)r~h!see+>S2Gz0b}D&vZIPK&j)J#Kw*}wO!_} zS~t=8u-u^h(q3PpRe_INQF;`OtQBn?jmRXlFFUCw1jhdkQf` zp7RP=zU(;ZxoP75*xayLU!|*=b)O)6W4Y)X7$~qexKxH!Y0O6;WU?5rdiHjz208LP z)#7edQ?H4Li)uFa!$wr0Hm*qdTi8CxZGEVV9y^mbPps=7k#jJ&d*R13?mF%z1~*6C{K_%o zZfLAJfN#7xxXYZ2CZI}72JC_R+E}rly$=?OXwa=?F5Jo92x~M$C^K`=WRyM!9z}>b zY~oNu0LyC@@=TP5gLow+(}`T)a=ezd*amF5WV+;__M`19O_B9rmQCb}cU?{JFwe(k z+B>#V3GN!Hjr&ziR?ZiCqiLi$JPW2VJNLWar*30|1Zt!hIEa&!!L=#q<5P(V;~qao z0t7)H?%vJ)%(rBeG0cYDv~)oXG!sNL8z(;g)u?o!QQm(VMGGo3<9THmT$AUILI1X^LX+ndUXiWLfR$HtBedBgN}OM9@PrzRZwS3y z;4GW``ac}{Jq!>o8?E@<3-D24>5#iSa8jl1ci)+`UPp}0l#ULyC`n;mlI2%cgd6h!-!|Cc+TPD~(X@l@Wllra0>@g!DCB_*ic(HtN6Sk~3_u{hks~M>k_u>9LgBN+epQyLry(=jg1K*s^mhH6fu4Fw62+r zP{l5;6?iWZB61=!Lf*UhEPoj6a$?=yXHcy;OOW8<{z!s|iK>%E^0L3DG^G@i>=`Rp zTIfTr7A5GR##8mN&^y9{*prxlddN)UV9#f{02*c zJR*5#RAlqk>fxvJNDY1~?)@>y2)j$R^(z;ZLfaKI$awhfSNO6JB=OQ-@_t)-)*>`3 zws(7q&-5Ypf%kTu5;Cf2;({@)b~Z9Bxa&iVNk-$lUtgLN+;-GtKbZN- z{kM_04kGm7V5Z--pMonxP|D=AL;jE4$!p) zk5H`Ovh8j**wVlJ!CcY32}1~K$jMQkPRPd5*5|~rnoqj(h16^OF2mDg!ak3=dT zz9w9EyKBYAiP1F9#+JASLkXn}{exh%9!UYsd=ile`0=bh`8TXi-X)xNsy5_i@FI*J z_d77%{T8p_ZC}yUO{Igl_jzhM?VcnwM(T%f|uv#H4w5z!`K*VliIqtCFMTWL5D3wi|BCN-Sdd%SP-N z(gKw+Ud|}e^oSopS#dj;8Li+gZi}K{p?+AD!uqA+5OD3F?gKqvY?_Ik_Z$9&u6Kuu;QW^;1FGrY>aBO6cuyb!0wDJ^*q-V%-vR7T+ zGW2<}8y-j60(l(oYfX=Wpx^G)Z^q08)!^0S>ln_LtOPPIy~O( zOIbv@ntE|giqJMR?_>y#onw+}FitBY%5pl*BBa}9#5|LZD8wPke$~P{A4{dN;^rZ< zR>Cfrf?h1YWHeTmAY4Kx0X>z2pra3oJ$!I(reHp1tmATL-M*}zd7KP?ZJJzf#J0JJ zxygLv2zWbx(<=sIUF@8Ud)5HrRA^~)?;c+I)HM7@=f@lTDC)vada$M z$av8&sim()f6Ia^W07q=zXGV6!o#x(K;bPe6*xfqJ5nc-e%k(i)n7;YNg>{apw{yC zy0*B4J-?{&YExcJ=6To&JITim7i#{YM#(txxbl^4Ynq&hS0Na5QQG+{t{A;KJXQp@ zPu>KsPCl{V6<6f_g3T0{Q+3+FAPOVu>Jt!DiS3S$pLS0-5{KyMG+8ZtAv{VLrA3i|1X9L8xb0i^*#X44 z=Ag5IDUr#7FSHDr(e{SA`6H

&DZ~!Gh8XWc`uuzB-&vDnKU;C#zEE25!Q_`cS`> zxaN=0J0|r~Fm0yr_B)lN+Qr};x$agQJLotGO5P2%M~7FQ7nB=mIn8j5%W-ke;aOql zoMwOha*dHpO>`Jl1*PwEwcF7Fn^)Mv!=68J#;|_R8<*5)E8~l)&Gg!}4tznCfWY@w z$zNak=Nr|gB24n=yos~4H$7K&Vy{vKiy1h%COf>*h_Z!Mothbv5fN~EPdEvn*qGa0v(vDuE9I&%KX@6Qhz)*aNhSA|cj=T(@W3VMk>c5=Gt6OC|8Qr9ziV z_$CPk(R9!9GpbGWLj1yQk|PPI&mPTqhu3pEBV)@@C>V{mhvy;JQ9bc(EYs3eoVG@; z#C>3Jjq@ZtD&a2E_nyfR0h1<&13xyE82V)30gck-u``(sbfdxl>NnQ+T?8ZZBLb5yM#q$8Bc3Y-S_AyPQv)jvgM*h!}R6a!$v!Awtp)y(_9A zpUYR{ZaaRKt52RX?`M}N+j@w{{pN#x9`*+pNZ^oX_GDOnHzmJfYS8XGojMX?EB z3-ZTBNC-9*?3*~8=tXU3CPPyNiyYTw#=M=?X48PZN}Zk3dOhFE)Z^FFO|z5WV{OjO zF~8IteA&S9E{cP}b>q}#TwoMHlP8ccTzzDn_(qyn?8IM4L)=wHl-uxAXky!p3#w@o zuCgoeL%a%|C|}GDgqIN5S;=Ih2&XuljI1Bd_be#kSb85xagCC5QE&_QFBqb&)5WcJnl9)*O~Q2dl`Ynku*R zfcq+j#D=p-XF~5viA-Gf$~H-NtCS71nWGU&&qX+i@%#V+SeVZ;AT=zktT^n(ZBgR+ zywLtRg>rJ>{b6VJ;B>nznP&67V-s%zr+l*mHL4!f%F=pugOKpm{z z&q;{4FZ<-W`t;)yh8&v!Z~UBJ50#pNG~~PWNu}1`0EH*AJBG{(6iam0u4OjqEW__a zyjr9nd*gOwCtnUvM@`<6mPM24b>Cy2neVfqM2E#ACu}7m4RNX*cGDT>A;XAc!+!rV zs$Kyaqw81zZ`$42vlw zBCLid!BhulYklI%(FdQ)sH3jr(l!_l=;M&`+N|C!_?A>ygCiSpxBb3U8;hzidX&!a zPKgVXrodUWHeMZrbDzybuIP6bcUSNOzvodW(Vx(?)5|+D2f13VYsl#fb1$DuKvw6e z4^66&d)e1v=1|Gsn&g3GUgvxnVeMlr41Kmn&NA+S^E&{)IY`*W_aGz#t0& z@oP~ze^ZWK>E{|*zp`5s_*Ny_Rl@{>zpzBZzKW`!x&nGykCPuS^gTrRgDkx?1O;Kv z|E8x8Z9QyL`2krwiDs}k?&ajn2A4-J@2KbXXLXOa^p*EOi0Xy|N#^TDnC>WlAwDWC z0;xQ5xJ80ljSx-H_6Zg?6k!{Vdb0Kb=OXretdsWH^oJ3|61hBhlX)A?x`u#`dzy-d zoE_EWDPw60cgw{=*0e;@o9ycuCzhqJ>LPsAL?<*G*>7Ecy z5w$7W%oo3Ee2nQYIy?F1-{rzKIojP4o#CD9c|irus{aJFQY_+=C-0vq%Sr1Fp*YMM zJn`^>p0x=50)o@^jx-q8{e=WWIv<<3NpwOA*#xYW_EZZ<{5ZCpP>*ogTq~i_3Co!9ubSZ@3_fVnlvqVJ_ zMD&j51*#2Z>o2kuTh_cOvZZf_>0;TpNmigvD4CD1Ml<|9Q})W)f^#<4BDlKvX87WkcT+zbYdorp9l?TG9@(l_u@BWghO;#@s!24Kx(uy# zM2!<=czq&HKjmTAdt`T?Dtuh4Jne539-BXDrIEh)=rjKXPQOLf;84F~jyeSw-}ItEBB<&r3td@A zOiChqMu;IfXwcIKkwzK#5<8WE=T79KCO@*A@QWuDG|jdI?49p>2d!9$J;qQz1*@(? zWT<@5`g(KCm_PlrGrd_3Np2RD`#ACqw87w~UG4mv+ z)Gr7;&I{}t%-l=&$mb4sBUf;PEVExDIoiQf+}72cp$f)mG*y4Bs05q7r;F(-OpK`n z^?ro>wt`kHDPDU!d8{|{zQr}Lf&r}j3hBef^ zwS2@%bIyvBm}J}?H@P1!j`cu*2rV+{6}Z@i!MTB3ojax-MY0h4w0URA(|^)@O_wgh zaY>3QZpMV4PWt11E*jAP#fa%8EG0KhAJD!j{3ojC(Io+j|FURK)2D;zDKSsR!|b;F z+q!Cm^vpg_`i2=6A?drH+N6!M4rsCc=Bpc5g-2xeldwtwq2+_iz9c=L;9q` zv}Z{|jk+iE(!d1(A2EBVl^I`iUM}eyL*+gL1@Al5+Kdor1^LfNA1*YH`!%HgwB=@{egjE2lve zv(#M=44LQaPHLCbV94PE9es-2Wf1E=t zwudZxw3@G)-OjSwlERM8V!Si->bb)Q!D6S)0(se9wXn*A>c^JnJoFTY`scb#u*e=o zX3qT7s4@DSe&j4%H7GT|Q7^QgEm$K}GGcP-$d;LAyV$<`mH3&XEYpB4o>k$1`hSFp zi4d0}1NSM9j`v&Ca~gk>>%#B9C)grZN<<^Y94&Dv!5`6&28MqKh~_%F6Iy}zop*0` zGUK+-iHX8$9MVsAwJ1gfe=59T_mREU*YGp54H~-Z&7JK3MS)&nGlVgkw$Ih$5p@3R z?R`%h*=+8FE~m;^v4*?HnkC8ATDp%XMF%HczGeOwjo+`176pd`-!NAa7{i9VmmlWh zrW+gzcl>tw9FwvmTYX@-#8&Ecp3UnP>>&;#BAbynr8n|PEn9PX8zfGDnmBf9!qlmY zYbCp0@}JRYw=Dc&jNEPe;7`5D=v(4 zsBF!f4Z4mr=Y2p}`Lyn}h_8ULaLLS1{Jq^iaJRFe`{b4%OD%f5y1_e-b~%vCG(fi2W;)SfT0ex=H-4$1-#}#HH#7W2rYc3hT@&SOeATss7spv1GBj}r^_tg(16wi&@ zG@PeWVksjBe%1MmB8o{bevsPsSuMvuy8q&qi}|3NVm~QzEDpcP*|fT^bzYmh$-tpn z2^iB+`cNP5j~$6dEG65)+?c%zcRL2OJraD$j$Mag+80ir@F(pSQ7;0IzOi++OiP`2 zkGVK<2gFgiv=UD1*(7^Tw7ZSbWr=naA#bMKE>mhRmZu)^IF2x!S*_}P?`&Fb zfsB5XJ#e`R!4<3BuO82)b+ikO)0A@C`t>p5xCzjeWHRx3b*ODEah7M=t7lYT?=nn| zX#*gllSn0W6h{I+LgY5XSZ8_Vap8;r6aZqaY@FszeWwPANz`NRIAZm>x-UlD9Eq;- z4S7Ql4^~1eHC6gHi_C$Ot(G6^PHKDdBYH6<&H3!EPB1m~oFo49|5s*=u?B{IBpgqw75 z3oJpBbS1axP!<^!^Ql})*yXl(obKJhx9Ls*NJ9u8mhqQ=kbTG06em1)1Fh&uaRaTk zC25dsu`0Z9!rW97TP9!T8tpsdZiE@flqWyB4-%hvtfRsW+NixAS zdU>Q|6vw9pXemi2zw|h1h^+XdC*m*v7-QRM_u#I z2O<6faq6475_ZYsY-pqcoSTnqhH3p;*LZou=KZr|!$4@+(%)3t{9Zma+?&ky{KP1V zVTxmb^{7F#!?d#026|RL=t7oPX(WE%5`=MK)s~aEnJ`N3l>H=YPk2cfNT3nsY!q_& zx|yOcLs9TygkfTk=a#_{y{cZYyCOC1ZqhAq>8=MK+4{x#^NY?a3o%M%$4~g;Ox6)@ zB*ANolf$R=j5q~Zv0uzTvD?KSEa=&sF814}phtznX&Brt(oj0ux^nuMHp{LsANXoG zXl2BU;#k==L_69U&c44ZinA|s7^`E>xid=j%CiJS8rkumZ`i0MubM)ynL3+pT<;f_ z-Q6ftc-pr&sorj;Ne=QPs}IN+7QL0*MX$9*fJ{7uX7cKEPLWQ?hxM5%N&DXs_6$roL!_h;RyJ(!T z2vdEIVCK1jD?d7&Fn(pZagw~feIh$lI}z~;lSM&!RH+I_B2Q#f2kT>&xd7H{dKzL) z`2LfMqxL+Oy+-O0rkk?Ew`&t78I3jLGfnza;KbD6`I}iK6S^7qxVq-V zt-R^IM4eXZ`0^cddsgCt{woJy1F8zmsD8{L_8k;CC4z3WVAO|x_V`m(nIgDe%jqrA?dyU&68 zc|q*KE^K_pE@^ksoh>>wd*7NpC+oluNdgpE6t45-)-1kBFg=-L|cZ0>JsdrmLBa zy9z~KLUJB;ERwR)bURABtTN!Fyg%v%W1kY;6LD02y&hDOSAOQzc=2>! zy3bhpoR4*GI7o)qNlM0DIIkj=lzL*yfMMcfq~AxbE|k*65UFmr!`eqBzAa|n=0FQZ zUb9TCu?ySnx&fWemO?jS6G%k>q^>bMazGN5a?WGPokZRzj0FoCMi;7NzvL1<3#qX? z#U=-rpt}ckD;X_D`JL7zjtDMfx^rHxS0U^43QHPGxQmB;R?Vh~cT~88;vytI4iUIH zlIidoBW19iE}yy^#*uo+XJDpaG)x~PM@V-NvoljEnCMr@#j|qB5Q#5OKMKe=E*bQ5 z`bKQ*U-}10dR8Ml*5NJw`c~{~{p-`)hmMQL+r=)_3)5rRz}xN9$cPD3!*~DL^8^=m zj(Je(>iI5D+`;98V>Qvu@O2%0btmq&3#d-LUmQ(-SMMT97~r1Q6dz@fj-U%y>X>oC zAWYl&w22!UjE^r1pRxFTB>07FOtso7EK5_t`h|2!{ba}6*xvdhaz*+8&yAcNiU~&Z zmx?;GYu+@Uc#hZjT1I!fqIg$2oJzvBD2@g8qa6pyqp@-@3%hTs_+}niwMgzGtIndW zv+kSH$(>kx|1=&M(`UggR#f`qzpR>SuFb)6L)KIT6`o2E=^VPg@+5)HH8r#9I5`bgN^q4MF zUVn#0(A&0fy{N*-n_{*qN;RjI?4Mu44D@Yai+=3gq#QAbxL+4uf_buIu9G$bE+Lh& zvlH6%3T*WWzFYFfvov4$s+l`FUiM;JXB@wWQpN&C2B<)`3v08E=(B1O_b^thn-3+b zVfNH`?pa+eQ;d+MWZuqBNf;K^zK#?LDFDwqNUrN+e!97qh zfe`AYQWmGi=0_6I;{|)vxPTyN&7nBWqbqwVW{NyMjd`_|4agvSW8^+H)@N_o!152$ z?k)>)4%1q%B-b89n8!H-{;p5NUM;0h@83pBO)5c>Dt?uE4 zfE+PMF*SUbNHE@rJJJ-hIP%}lqj&TQsqw$!hq!*80x#!nQDn_~$NLyAD!J3ZO_~+y zGyy=leS9NrGVcd~t8uMd+qXC%Gt(Y4{32$jTCP|vjoTtTYs3=ZqgJCukACuT`aav; zQ210~LGU2CuZmq3q2{O!YhRj1zX*ThQ6TEKQ$vKaK0wy`z3{uJ=LSe~7)zEU@B1tyNn9wl9H|qi;zABq;uDAH0Y?y@7{jGf3Nf0 zXs*njm_)k3zj$dz+Bmx(IEd4N)}&=;Qoc?{8X}{(EZV;!C0u1kN{OEFtt_1w=5O)( zt-ld476=)DU_rq-z64>1auW5Ms39tu@^hu`t4Tspv#&gOo=0UOp6qp8ufv`ZS_xmz zgq+R-{L4Q3{nxMk$aOq9@xOY00s^f^vmL*7;XGbNBlT$S3c;<$P7=+3dE`*w;LJCn zqISxP{@MSCR{BfIlfPo?E;7C&ix{%lDw_B@c%EI;YC~!|y3W)|FO~yAV-jvRHHREt z84!r8&VNu;ey1}+`>zDbZUAQ&mTE3l34 zt&nK1w4L2G{$>_bqLPAV4b};L6Iri^jV=l(Dym6dE*ZR(h=&Bijf4xvRPQz*pC9x`HLV+7l2UH_gzwj96!hY zB#PTD2Sd1EY{PlU?%P!j_;*(Ef6E&6GZSTwP3WEm^sE`mETSwNBDXPuXCkCzxRyom z$cpICi!`)>-AhsSIMxbG?}a0rVuJe-rxLuK{3ToKmytDu^b%P<0kRHtDp!BjzW19p zF$C*6E6G&;Encm0%Rs@$?Dr)A)>-HJ=NGc&umWj7D{~0xz9%kZF6=KAdY|>Z|2D)k zQ_jzc?9_;AQ*NnXYf2v3rR$f(28t(nPi|0W|i5p^@8auEsE(bwsrOi3w+GM zTl*bAuCdhN{3v}Z+i+OOe(oUYls{NQL^H+i@ zyG0o{#T(~(7~1DNnVt|e3EKaIxc820vVGnKRgfZGl-^OX(0fNf!9o!M0cp~t8+r>Q zN>{od(h*UL^e#0@3(|Y9p@$Me3lKu~L0^1--*fhJ&Ys;pyZ?rS=YH;c=9;-@u9^AP z%pg%vcv0u6*wORByZ&k#4E0x!JYEw$bxprthhU0$F@aaiJPm1Q)tPZvt_U)TP7)}_ zt6BZXDUqYLO!V?u7wft^O9Cfi>>FP2UFvVeu(sQiv%}_dZ^*K%Xk)U@$7oYIdS?>x zd))A5+kV%2LyzA);X;mXn*N24evXS5PB|c2+7p6Emh3)WpKXNi)lML>T5U2BC#}ex z1;V!7*#LFa8co|UnY=6Pksv&Nh4k)Pq~KD|^SG=UkbWB0zS+Y9e{dUnt?k;XT^28dgq40;)r^*|rp={Qn;$NVg)cllp1M}wdpnHl>Q(kjrBpCZ zLKS0bu(T!_ldUL?6Xa@7+fAspB9XhhQZ&9j{kz&yv<*#MnEraj5M$N`y7po8`N-6? zux!svEeaYWqt2tvDz{40L6$z}APyPqimA&k5iaU3$1@bc&p?5HH!Ix({*pHX4Q5+MQ-R9XPgS?;cK9Tp{!1I3{ z5g_vVBTzz3XI3_sj`wf8ScfOglXssX!4NN+! zz8h_OXdz95p!8=kUL!>Dv2o@E+(W5aB=U>dU0 zT$PU)=?@#{d-Cy82^#0S0L}e}34WjaDiJV7kVjP9_hYoZ1J-{v$yeb0rbH~ePZ!OA zny1C{!ykL1^%(CGJKms4{muD>HN*s1_=AX z|4B8sxIuA7>OP$q0(|R#%8O?ZvAy{Z!|bh;9d=p&iBj(F+_u)7=_;)2q<^Z0AP-Uql9GUhxA z&_*6xqQdq^KYWL&MqZk6O~E*J6FiivSx~Ug?h6w;nIb^xd3HZH-0)t0vDa8yrF7hY zL+K`a)@gV7zM9Y2fetXitl+hC0wWE+<7-395J{e6h}B-V)A7CeKdZt$Sf+seu5{=e z{B-u-G|Q7mc24^}bsi7*-VUS{Hk@?O=iwpUu4Xg4Qfqmd5Rk`#AFe5{aVC9c-;SZ+ z@FToCC%$N6h)Ka3J-mg%%deu5{xTnf4(wkRl4Io|?fG^sM{)&dDmfVEIyO1IlZ^ZB zIu;%AxT%5NsOlO?=G*3~uedkWYlibY;OQ@@u|KD2^5`G#RQ!+7Oa27bYz^CC7l+y@ zz)CFKQkX6iJzVlUJ7cq{*GQ=yc9S3#F*EI2Tyv1N7$8Y_ky`=9B3<&Ob8oyT5dIF|NAb9Eu$CXKDKLIHFc{&b(XeQa zocXiKe;y)7KNDYhzE~ZiJ=y0O)=9_rl?{za@D+5|;wolmXSPR^ z1g$7(8Znydwez_Z1TkAkPP0Bbg|+AxvnXc;XhRKlRiX(c^|+FV=4Vdd_FnDtd6bWj z<7h~Z+0m?Zx9^Q`YIL0%d!6Tg4JjahsX@Zyi2u{wr7Q8+nYLS-Kn2gB8?OTWAB;C9 z!GgS(Glp=~=R(OK0?$jhkGopdKy`u_wKwAn#}gxFDo`&f*1KZ*zIe^OtMb?=dIlPo znL98VcSTnNdpq%aUn6FIE6_{3GYu;xg1FGB?6Jm0D|7as&RKp*{T zQK)%g;lAp}?Vl6*w)J$=K)>~6-^iEW@y|bU(f+@#s{emsMr^1R*}iU7YW>%v#kb7g z@=naO_d4rk@*I;XDG}+(CABrH`FxpP>#eNHBesac5D(pvl*;Owl>>?)_iRvMT53a{ z54He9gw8YdlQ2Rpl~o4YgG@YMtT6pSquqyRqH}}BGaXhSjTM;5=RfWbMy9}?(?im+ z5^LY4YE|wH;9$OY*2Aig2D^X<`CCY-js!o3m6}mQH!v!QmZXts)|%iyi9o-n|7>e1 z(Wb1YJwwJL44F#eGk!m78|cV^w22>N=EXgSnQ2S_gEB)&y5D6>P(umK$A0JMkM0fj zNT)^pirp!@<;a+MM%??Y{swOm4?VKy%a(()b zc02ynw>K|hNX*)2xksYZh>c{ypbEdWpUA~*OdxDsxfmF~Yr5AhgCgcE!_R_B(W^KY z4%zSB?^?JK_x;H;LY>BI%+|Fp&G=z(g6il+>-}K+RD8>xGTi8JL*0T7AdgkGvNI$` zR)c1+oCVE0DKeGydiA@5X~-Wx)qJ~l{;AgbyRBSo2YdUo>oefK$}QFrK80!Ngr%xk zw49j_J1o5cY<(az-@XyB8pQkz1&WyY{u1Ad8&AOe*mR;$$=hdVV5In#Tq=o=nNnV^ zEV0}Sr5>nYbbK7en8%^)RPj~rqlB*v+_ZVFGtV>dz`%2p9P8DpZjY&Iy#xI%v7Lk2 zXDCbx;)U>p3cZbIh8BMv-(3o|F}t=LqlZ^!qL3dysj@{IPWsy_z<^Y!qsOH?I`TljNp9K6y?VC!_M6oc#TUz{SL`_+qI zd`G``mwJ5+XF{#!azb+vKtlU#cj(iVXAC&Y5BL%qAISO#qsH~wOfwBhSn1%QMNRSu z6Vro?UPSe%YK^3+Ir;RsaOo(~g@--oCZjg^xt&h2aZkdeoZx(o$6;u6SIZxBkQs31 z*obW%bjdWfzg~qnuW_1{o9UJ@w6%<6EG}v5YsDPs^Pc~Fo7LN*{1LY`8b>3*E5V-L zv+^`Wg|zw#-Gztg6%nVpW^fz`jhxzb)aUR3>ds6rc zG4JP}--37*71F*Q{Hb|djXhfrtJM~?Iw`(P;tv3q{(lU>4wp*6D4C%eisTDx62tg( zq2IofMV9cH-^WbcVPdMqXH;%!1D)wKEiOL7Md1|pica=mc_&*@3i4NGf;{)xbJD!* zJSuP-#>&orP?W{?41B~XARw7p&2w@T970If>f9d#3N5ush&kCNZnD>1{z5{j21eocrn*8 zG=-z4 ztu&c_yvoxE!POtP+bbPsMk$CTL>~Oo^y5F4-~T>O?5_^w|0Ym1JVsG{GK3b?CK6N9 zq4@C;&B6w+Nm4Tfv@F(B12@gV_NMI+jZ63O>COQccb)X$H|Rq)|J2-e-ut^ONt>k7 z@8&~E9M4|W?xYphV^VQL&QMQt_rt4tA&!KMnR?;xohO;^Jkyhyb^%kX0J$M!=zNzM zcCn=>oaJMwLh%Pjf_YYt*Ue_`HvKv3H%-o< zjZp(R!TC9k@z9R|ca~W>AGcmdraF}k2uEq_p@JOdn1iwhwuLC}@-&qOf{*nF{fJ^b z(jLU%b8_JOM@Rn4yUrpW3$Q1(6N|8W^M)pQjlWRSQ*70_U&)9zPrfgXSSubS#=ech zgrNH56WB<7ljARyrq*Ht{Z`G{c-0zDtkcsK?4Xx2!D?l`PD|WpDx?;71`-hg20fh= zh#)*!a|bK^;Io;Vu7pJDkaRqI9D`3o}!R+bB-I zcu!x@5|id6D2)@kT2pqQ^UlX+KztaRKzWzShu zXIL>z%RBm$WU#@Lio8XP6Cmd{(+rReSkx51YRbx34YzoM6&BOt2K@su{AWKCe+#r9 z`EBil*zf=Q;ft?L2kc_MEdaxg)_+rKo$&9a^Y;t?KfL`qTq--&uwloEko`YT6w&>w zi!`&qt2aY1-XE<=Zpg?uLZ9^ zX>CU~*iz|;utW04FRR~Ln6zG!*TWt1DCnl5o7jV9iorb%KZQDG^l1vAaAQnxOgWBt zn%RBh6u0aZq}CFVG4M2&>un6U=fzu#$W3szE=Cht?BjU&PHVXv7!~N$O(8Kuw~3qF zAtOSAP~*m3J}+OnPo!dWsN{~kDj@UmTFV65*QzO3HRu1VKk?>Y^Fp9-vRbX!>3bvs z4}?Jv;Os>w3z`aAtl&v>fCaU*^b$s4pZNT}E6%W^J>B0Z8>$+IKD7xzIo^|I=lWA{ z*us7RLBJgC@eP%}PNv2(be2^!?wX^X2t62^%AEgJ&FZk?@kj&2Wv^vkg9Q6_d3r`_ z4;c9F&&e7I=czAJ6 zg8Lz0#qyrldl*ySRKwJIj_+<^;|4T}fMJ|v7t-aHcFf9eHE6_g5ER1QAzs5{L$=xJ zjD>JfCB*k(tB!dWKr*yCg5`wj?e zw{={UwR)66y@%h1o5~-wyYcAkR@8dV9{@}KOE4OPa5^DYq0lsCrRAixSLComBHmzB zCt^*MRJftz{U?jK2pY-CbT8zpR;G(|8t9j~q6g5&K4c?34Gf`nUpDn}9*z9i3Zk_% zm0C?UyM3M8d4LV5aOUS~@Bc?oR>RIzBH|6e-al-!nDdl%ET+uD8bcPvl(C7FjL@i7 zf9A`U4%`XUhVQJ)`ROcu73egh7T9t*CaS@jqx1qx=ZlK;ukQDNPlA5Rr(Fj~db3kr z9UDUdBJ7M6K$lQJCKE3It56F-AM1D2&v<|Q+?A(mdC<~v|DzF|$}jy2=QTqaq8Ue_ z#ulA6U{ZFDiSd5xi}nmcMx{`%!oy=SvL=*%H0Njo0JB4eG9J&$mOGi7v+=S?;Iek^ zp6-(Mc?0kLyrO!}FUpy&xg0wZmcMZdeVd{nI}u50u%@X@<3d2)hjYG03L$qO_@$go8{0Xc?YtD`n+u>`rmB^>%E_y z03^}<94z-%`8zHyHLQW1=^cs7MW9)Pt94fzGvfE)DL~HH&)z(r=KyaJes=?m`SHYv zGSJ`%)@82$R_J8L>eiaL15?Fwg#N57U(-L1#@t_C)*mqbe;H0Ye;>>LFqkOPd_sWI zJ;j^f-7@|G(SNVnw_0VeR63fnVzuKZ`-K_b`Q4S*P9uoS2g9WDQ3Svsr5OiFZ&c+ z9mgu`Po2-b*-q7OQ{WlKX3*IFvryG5a@(V^L+esIjk`Y%jBhu-L9E z`8V7J7BSasDE`!i8=1wRO;6fq$3sgj zqGTXW2nruaz#+Ftz8}=6+Y$$| zXmeK7J7Oo4$wL0=zv5f|@7|f+W>s3vUQRPjg|r%&U3tKz%oS++Mw;!7^N`p`$joKS z)zEP=GWwtmR|rIrnos&;A2$H_h2n}q)aFdHz%TW8avD zw83b64#0J)&ylx26v0YdRvLc9^$yvOP(`FL{zj`PqW@x;mYt%YHjt%IMv>gdvhU}`3~HO!f$8L_BmBsZ#Z{KY?i z!&i%^?STmPfKm#HI2#CS4C z%Qw;Vrq5-OW5*P~oy<w*RH%5S>;%y7-S*Yw%Q|Kkl#ZO=EliP8Y}& zTrB;4WfMrccVQZS#r%iYLuCrD76W{RV&4kGyTgsOMycp;*TS9*q--xO<@% z!34+h{Z{3moAY?lZsD|!)`n;cAXuF&*YQEWRkT_toPoU!ZE!v z_Hl{X_~G19>{l~WrxVhlHHkE-a+v0{v6Jdv?+BiEfuJ~*t?8t~6X#}wpIhjLoz&a? zDuT(4Laa|5hf<)&6L#Y>IJ29cc3*0@CZ$Zjv7IdO1nag?4vvm1+nC=}AuCLeIF~1(0l3{HK#d`-Fh=m~7vE9(zP5FI{@+5KsDr+fo^P@lt zX7R}@WT3fW^{=nEbQ8Z%KIq0GodL~*dAs3Ns6jn$S4???z8WJvd|)&^d+xo0CaVvB z8tTIx&D?!~I@gsN<6lF4qij#C>2TX?Ljh1c`iXU0lU&cVq+7RFhKL=ysNpK|8gI>R zaz@H6$N&M1WHrl3_He34*B&J$AzAm1M<5;d3tF^^L%@}iTFrrSc0NCWIEog&dW4KJ z`tHP|qolwdwtj_#5kw zJ8am?$708T=vQ$2BSn5&-BfpIp@+* znSgE;eh4q7b9kds6$mECttCAp7^fHgQCH<@`rWVjZ4M(R7~Ke?>iM{-kkTAM8@bw* zj#CJrEmV7pKkzO@lXcyi$nv6;7zqDg8rQMLnJpMkW?2 zHceh<3pt4a;j(8RpkFzid#h?glWuQA z`18h=3>{r4Gz~N-&>%gVdRP@AnIe>vxjDMHUe~ixKVH{$v{DwoSH6-UDkM*Y#8_z^}_9Jve*m@ zSsE{PBUI4NNukf`=b|9rSawSScbD9sg=c(k)+0bn=*mvDA zLygsCY0uDjk zHL=t_zwq;<6gf^9!dEfcB}FCDWg|4lp7@7 z1;3Td`=m6L_0cj!H9LUUx3vIm;vr~zKubZtIycSmK!q}igxx_72EcVzke zdJE7}>AJ0Ld#!Pg5j&!qhw>uqd`YH7rGoQ)o9}OyHPa)?T~Gy4CUh)topK?o3)U&V zk9xWkd#?4UDXT|0O-lAz`~rimPYTSE`vNmsv%ebU2$=<*SFBq72?T4vEu0~*kFyjD z?H|xQl)Sqt0vDXRq^i;r9~r-ZxVRYP#d4!^sP|QxvsN2%N{g3HtSvwK(B;Mq1o=%1 z)2h{k&S3d+%8uA{?%|`tyA1A8d&I3-jij=*yB>9yobNf_0yTn=L?e;RuL#GE)1fP! ztw66*G(k+M?>(=%_0{P(t#(2<`lDk6$NSd9Wj6NhfbI9vPkxjvklJP%!QC$`(<;WL z4G_v5$);D*=w~#rm(t3b-fkOjSLfgO^44pwzo=$I)l9}?)CJ|vg|4p4pyiCr_6>Bw z{VZ2b(HRrk@#CoVzEvgq%T&+#3E`091L#2GUEP?$` zTk71xT*P6KL|JdgK-_#DmgsUng^;n{pMi%p({+_GJ0t?fpnTGgWs?wh7P#8Y0dr2c z0*8>(P!dqmj>N2duDWFLOT?M|q@Om+e#c17s&BtB3}{CD6O=+(Sg^d*aFhyR=7fu- zejbdM5bbA@(<-u^j9%QPB>1I6(@%UHtnbw&w?-CV0&`2EpkTr7`5JM%3-6NG#d<;M z2V?WBiKSkMzQ1iR3G#u7K3ebiK*mIynz;s-n^uZ=J{f)Ta<#lTsvO;5Q`}F(^1~q8 z`I*#%i+cPfy`z_-2pAP;d!j5_Oh5H$_C)mP@euSBnV3$eZkzSSY&C8Kcy04`F{c=G zlX4q}um~aMb1fX^2TQu*WQ|RiUD>d<&EA!xq)(=ia<>bRLOf|Zx|@fO6aDs(+9CbM z%Xe+BB<b9C#kv9}iu3MUu(3u~H1VtNK8w&Pt2 zcP6IS0IGp^Q@M0^jMrT^*E*HS67^D=_2u!~1D|4a$7l<$PQ;3Xqd&VmV6!(hl!?AW z&%!+6orQKL|14H75$(~>WGb^6r=;(s?ig$R<->fAcJE;P@Ye{(!e6zd=GR?PELM7F zs~KV*&7Z@zxfvGnfJOT-^6}-l1DtGgO&U+(V?Ex-@hhw5WQ-pc7}~oyPO4)LCc#m; zWjsP8wB1o1oA$Wx;6#XCO z>7LPHLgRp?&BTk_7l@QIStSRMr0;3DcM`)Spr zhHao(P$m3Ii#E~9+&h#iH?53;lwy{Oo?O1Dmr<07LMnl;NHb8*QqBk3bpZMG_U2h@ zQz-GH2qQHn^Yn<9zxszCe-`3YmqfE@KZ-_&r|`f{Y-0&pMRp2p)@{jGba~QOaM$Ja z_-)i0!Cr0)a)R*mgI$7^nqVEdL6_&ZOdVoj(kfgrAR#2vxRESJ@X%;~??W|Yq7BPK zh5DT@#we-aU5*CS(PiKIzGkSr z*kaC(c5`O+(HjeC>Oo63t$z$qWXgrsq_yWZv!SFns)zi>v%U;TJ&~2L(w&gy88N$- z;FzL0{dj#hJN4Q^>^*t+%I;f5w3f#0>W1S(=No0Lq*=Hf)u019Q>Jq%cyzN`K;K8cPk&VyN}(Q7MlUI9kbXPD+&Ny*)8dd+*kgYHXB#;I_-~ zof7lXj(Z}>B#-7*FZM%y#%3w2%nmx#LoG>sc5>7^yNSEP>NP93eA>`?q*3o-__bTF z0Pfo3acVG9M@{D9#gT~nHy0c-)&nc0f?ZT)x;!@P;e%tb42bWZsR?=ZsvIMfHx;}` zLT!JFdD&S8rhzVvb)TZpqI6Be{zjUsAA_z2?tymXUqhHv z>fd1e7Gou4S(j5D?nT~GFI1zK;J8QmjnlGFdO|WrjrKfwLc`jREn0TW?UuIl@8^@> zUCoj%PG2QtNrn)|(OO>4kxi$QojWw=A-Cv|TW}z6TcF_5s4u>o*^YL3>f+7Q%H5Xm zuC3vKz=3I3Cwh(q{l;}_7buusw@VTA`BXl~(&x0M3}%s=&e*=K$3Iw`jYWsEq>DGZ zQ`k<<>F7dR+9mmO)5tLGEtb0e#GMZcd>rpkZXuAdQVsR4g@ zT+hn&VeJ=s$!q(mcXkYE#9;Ku0zR{I4Oun0YViHQ z-Hx<{b;a;Vt1FMOE*s~{Kagd;{g!gu(Lscjw}D7JVL7Xzb$~pvu|q2KRs`LfELppZ zgx$oaKuydy?Mdwmp|<9$hm*Chp39=HYJIu&y^j6u^4kUszino?(EFA$=Z*3Re{scO zOENL!BV%2eA7z0wbySfcFiJ>$pU?~SCAqIrloB}}!@qG3V|gBMdNrhO_)?{ysa(9d znZMcw-eud_D6O2>zA+Lk{55{VZCv~dk*BDeZi$L9u5D55K` zOAk8D#|<2qYG!x10#p0DKlZ_MX^FrD&yj7H06Ll4yb(aL!jx zMqhH0L6+MuMm(JKz6*4L^Rh2@1mB{x$**uCj!bUw zoHw@pD&eR7bi|cBzd0w8j8rt@=z_DP4_a#es%4ElB~#_2na`Y%ft~Z#Zx6hNsu}oz zZVsNL|47A14z>54VxQ&e|S~$|as;=|?^`#5o<#KGcoh z1k^>p@?A7+t)SUML#7Fqv+rEY5`5p{tUiN_+^hF|Xh_bjAO1NT3-q~(<+JRgKxiAb zu3tvDqU5r=(@5UtX4%fWZ#?+u@fzGJPbxqBV0t=j@3#nC^%S13ZJS;=6#xX;*Hi

mVb^28TEer>+PlR5W**{F1M4yhhH$wWDseqh+XCN zXM-@MeWy=ks~>$pt3an1T&6zHh%s*g>eI5izG#oFJhYbi=xHtsv_o=23>6FR=5l13 zWx)BC3BTkB^!Q*CU-|YYpI)&&_CP#Cktn@{hu>FnQJ0lMJ9 zM};)!>h0U91mD3Akq>#;8u8^|3rn}Vd!Hzgu_}*5>w$uh`{Vs=`{+x;glb0o3$ue# zFJ5aa`qo8jk}sxvWg(ieYlMHii(B_4^OGQts(ahOoAnx^rw2XleGJk>Xx)-lKVXF( z6w%y;bR>pZlWy^zT8K~Bpi;)gqP7YG3qE?am>VVo551wIPo}Nxl#!qK;hzbZSyD!V zHNq;(zuNdL_hMvWEQ!zabDPxL;`+<<8xwp8bJ{wuPMN6EezKh{~@@2!G}S4aQ2n-=AmAzZ06E1`CL6Lux`oa@LP!T;nG&jvSwJ*c@{C)+wr(4v;PLMS|)|suO@W% zNRqG47~pe?-KbNL4_A5J%&j(=ZUX*5m9<>L#}lX$foa>b32h6>!vFl}da2**-jzt= z8Sf{B)C$e^NqJ1q+ePo}#K6B8cjvcr+I}&ke@?2AnLHV=rfofTvSpYN!}>j=kmavx=!k$U!}edFSYorkB<{L8P~T-YO1?<-fOaved}sP$URJi2 z{P*?oACaqm^a zl~}Xm*3hpOJAVPD$v5)`aqxQQ=tX6fr_rqkV7jb)o9oh>k1goPg!f)?x0Me{P;jp{zMW@Dd1@^1l@D~zf&p%p zaCPK53ZRkL3WZ#W7@Ykwd#yN1_J=oud;Wk4z;ljl-V}UGR8U=GqL%ORE3fQT+(QyBw+OV#S2h*Bi3^8BSs(R`4T_|;inP-!NJO|CJ6^!NRIt*4M`NT- z!-T%%c+xGaJ?15#xB+#(5}Ivf$7Fs??$YaY?qLyPP|#j9lt`kRxjl@K%w0%>dDaL% zGa)Fa!hYb&t6_QMm`!MK?3fZwOUq46Wi1F1Z-C7fZL5Uwk*uq%`)XCqd-Z4e1{~l? zTWmOiI*VUlStkN;y@!`;qQOP9KrDtLGB?Zky!Ta^O34!+%ooL(ottcmH=TQ&Q`kMX z%Et(B$#3Gk?qwFRwtXPuv*Dl~c20~~Qw~Q464vzc1~WuNU-C0dc%00#mf+_0_;b!H zrq9oGuPkWEGMfYLD4(O`wzKzAsyP+1L57PT$Iov~b{?J7z)pHN2@VklUkHExG#+`J zW+Zbtwvym3E;K!A|8^z4#z}=<<=IjV-|nWUU!K2qAOXF^7votkpSMr51#j>^?{k{DK~2t(TQgl6S6Rmy z$yS&J#AYRs7qjNK6;|LGjVwJXn~ZGcu&XZ8{GwYMS^`8N=I7@cGQ<{70>I| zWZvS0v-&ew%h%SF+UJ?Kv%3^t8%@6eC3SZJy%ELZ?4N-4ApaTjwrvraVsw5K0zSyT zq$V(j<@9kf*r0Guq@_MhydAE*KZ?m>ImtzNX<(Y^_c|9$aojoiYIP z0MZT_n&b<}4?9?l2SwugIpnWu05$e#IArse`vEr3)KmU(lX4=w((#mZa7JS3wEW8wnE&!H+EMP>XEwj%T;y5GRPl+i+b=6l z4gdAMt9dHMs{R{b>tn&@bbyznJA?mea2^2#Xe^nVKsB)m)WS*L=(WnypX^j3@4x#sNGhz7Mg0QK=IQIcK_L-Dlc#VvpsBNL?4)%{_Y1lVKRL)= z4xTy9QJcNv^1rh*w!9owqZ!K)ke!xBTiE31Mc1JGIOSgJ_2j3|1Vr0gvBgfBk zMuv|hgOy#4@t#*n>ZMX3(H!X5Kw*LXnHUx57|3{muU_v`n7fn7k)r{5n0N6#g_+MS zSc7|kG*Y9O%{TufHN*ewDIX&c9p6p~ufLf^?eEr=Z8+lqqKu+`TYzmCkRHA$FbqU?_IqkUaHU_F%#rG39tvA6&P1FjMrP6jsgVPuqNc& zQ2qu5nUfAv(o8FK^-1s_ZiU**zqAt#Y9(LptlT%_0FMYFOf5^orBZjA4W@?gD(UBzLywkwehl6tH1WZMfh2%_E;_PylLAW(lBW7ifLa9QGq|&HX`*?h z-Uk7EJ0-G$@%o;I1Dv?9_dDJc5=NxsqPLQ3*xEFOOq zYZoUiHEG_CdR7fE>Y#RQ)m;dHCgg&d#@to+x>Hhvqph;XOebyGw`6r0`mwUh`{ds! zWDJH0_je`jyi)f9S*{;E+%%jyi7fi~CV*(9nZ7R@RWunIlWE^*kyUpXU%NP*J`zX< zJQ1*c$*UX=+i$~h>xW^l zTH1#o$rs5!ey{XK>F$-YNV{|v7!7UCTUT|thsE>SKQe0Z51%szI_An@kB@+n$5X96 zRXUXl9*8)smwIij(`u=ygkq&TRs+)=KK5!TsX`3S5fuk&T^FyK51SH0hmto^vB@Jl zFq?H49JNAla?D3=xW<8ScPfmd%wfXQcMcr61cUX0?B`gMD-Uo`w~m5ov^Ar2PfDIc zlVIb%ro1~yGkr0h*Klr4MlE@7-+j$*;zo#(x}tgtjHwl~AHli`usnO{6Ii0q zc%z$kac(qPw^AjCr)?NCy3%tCDMwdU!ZA_juBt09saY_6%Q~g8}#%ty1v0jn(9|HA?m2!~{i(}hWyG>EwwqN&t&&K2A zBZW)(E`-Wbw?dwIklgQUzNLK!UU?vg-N$~{Q?1|JJaUAxYo?ZxV<~~lvLp)X<@7o# zyYqM-x43aNn~mnK$>_(8gX>*_sJ%~-uTY*Ju?Fc4@*7@l2*g+RXXD*SYMRU00WY{H zf0~)ycejwUE9cqkaKU5Q^FxK(&G%T-47pXK_RuM&!zP2IFtgf5bZoYtWV3&+);8go zX>e3dTn~iCfR6o=u!&AF#pgkUgqy>*yJoehrPiqyQ z3jS2wIYboWVatKHk~GgOtOeNwD(qO8?>(_ki=GdzdcUJuEl6&f0f zVc$LuGZ_rxs?B*N8KqWEAZOk@VBaPU80Jo?225MJfblZNGK>EQufWTPB5y(%EM@N1 zel>{a$n=aCbUEfM*&jOb=6h)-xycG|m0Ud2{nGm=YWV8>1`b~;MFd)Hds z&)s(?AYn#^Zp+GU3BhosSMT_-Y}7lUOWz&LNs{Q?lR}s0QtW*(kz5{)%=i#5fB2=I zv(r_?l2mxRrR2d#BlY-P)w}Xr8=W4*c^`xRpgYGIywP&Jc7o9Gg%bH#If>FpHz#w< z*YZs)FVbY**2p3)-NJIFAQfPO`Ahl+2^c7mU`@6nBVDJgUoi+n+_*RnxZy4otM5Y1~=D1PSepBz*5MEX~{f0>&7gxcHvD|Jg|WVa{76xwH$# z?$uWHc$Are>yI5fGit+x4Z`Jje zS>6uteq#<64z@C^f0;VdK%@8)z5A^NPLWYZ?i%<+R*XP^GG|1vv-=^kt8qmUH~(7o z&KilZiCT)3=g;LtwsQS)|Nr)6^)-noMP}569AftLqvxU#pZb-XYWzej{K>69{9?}^ zS!pYo|6s8-MQiq>({~O%Z$P(N@?tb?qledB9%>Bhth9;c1*UYOzzJ#uJZ#p}{n?%r z%ZZ?9r!+<)o0(chxvQ4V-1^k)B#PV3;h}R2ly}D$-b}ML{1kzwqNX1}cN5T%VC8=0 zxN(fr?p6$_-|7MTVL2$D+@2~lM3n0(weGGzO;7l{{A4f1d%cCe@Rlb}kf}#iI3Ac( z*GGZJD_4go+|_(Y_+N6Aqjs6DT3Ya^?a%Fszqj`J)TTLFmWW^$4t`Rqu*~UQf||Qo zl{z3U`hiV)LLg<(0^4mY5A-Rc(@c#4wf~gr5Vm|PeyaGB2VY8co@Y7${r29&i4}~U z9h_5~^Ft_^H|N!&IUO9BgXxoEU-@@7boqamxOv5LPR);+Oqj(!#eJ=USoTj9E9 z(bMm8hs|>ppx)QH{d|7+UM5Jq;0hC@3QbW{+*cQPnznN8<$y?xO*395sOeSNO{If+ zRt@=XSPQ2)HN4p%D&x&-^V^4l8&9HJrvk17O;aWB@EL@yNP`{v>4V z2u)Ah*SYd0z!ap<6e~SD8KU7za%g_x?Uurir%F_Ga&CLu3B=wrp0m+KFfW%&{h3B+ zOP*VE-_3exIj-31B`+3d_r5ZabBItnw5>|(WZdtFm#R_O^oU+W<1 z=7};NCB5D(tP3FAeS{UheS98|%fch=SjOMrQ$@BQFY4iz z2MdUnkE$AQtW+DNt|OaCj>x`Fu_g~PdVxGr0e?LEm(BStoj7-Oa^{nFwOCKgP++#@ z+J5RQTyTWg9N;6}llymBNzVLbD-{0unlq-=&a^xufu?u%*<}$XKHh;cQ{GuO3biU7 zCRQg_%OZR15yOQuvo|y#WjBuP6hoamnTHM+VYqfWOQ3viV}hSjzcc!_|M-5D`Jx@H%RlI`(L^2Q zo6mEv*ajEf+Nks?@e#h~rTLtyeW9t3gISzqvD|hkLlNnqTnFfS^DPWI$-{H5-wO81 zKfG!f)tRvItiZs~m3;fb;mgift}+cd?hqQrKz3Q*!H=jZZxq%{bGJA8f9~PF>ZR2@ zvRaXKEi9#I8W~qO9ITh~i#IHF2r5DeIu1{a6kr^cr^E*g@3HpLn(3zo^*$ELc1#lx zY2=HPAfjYi-#qA&(={s(`Z`tO?5aB%V*&Rw_$9DRvC-N4%Za%?QvcIB3$&*J_dybw zjFXhOp^=CjcwYa~_dO(kk_aUfkjuS>x2J85Y>meMkf-e_d-&Im0gFyB=k z1t|ujAn79tvXVc@9(7chp7{8#r6x9P@TOJ1G&Rw9QaI0}S$jxE zR|EKPx*I(nlTBmkt}>_S61t}P*QGLB8PtS~!^xQAo#UNULYrb*gt(8f3dbdQ+$IJx zdyxUn&ifoWPiisB)KRrpn>mO2ptQWbu&Mz=3B1C(-}h<;)d9}9$UBa*4AZzbral1< zSm3Upx9xCfcr>S~>wVW|asUruS7jWd3+eK)8SQDO%($t+^la9C3BjIh@4s_WVu+>Rw~~Q86zK%ZrEvE#^#C`EHT?Qd;^LPXgDsfZ*k^-FOHo% zehPxJY}s6gjLV1|H>5B(ml=;fhCBa?P3>uR<=2<>53$fcqK?lnsbx~eZ(W;)m{#fp^=7j6??)z`NCFgm(kMkJM<8@|5?G{O` zcdg2*8>w zXT}*s>vAA0k@?gpTfe?j8)?kC1zFTmu#-&9V?x95npiHC4)K9$;O2*>G*;n7qzofY zorKkLSdnJpm3N=OGN*(-u(r1nOcJDavx+*{R^bk+ha?pPKnBKN|R_iFRKu zGNX0ZiPE4*mJ@1_ka2(9?W^{5e{Qvr+i$%mP7qB=83h`2#b_}~)U-FbY?$*XG>S`~ zi}9In>Id;Leu&5UE6dMP*Q><5!t0JJNp7?A^cG5BDOHr3hwi2in5w?P2N(}K zy!d!r?eRlO$xYy26$3i8QD=mPU|=?R7WolViUo#>k%1hP5wvn;2(@QQS z_eB+7j3+$ajlRwPIz~pdz&=g9usn5l&?9KCMn31wCsvb+^=~#Q@2;|+*j)?PKd70i zVmQSCta_B9YU?I54t@RRc~j;FIaMxq1|OeEOXc)(3Ds85u1L-xUMh{ zQDR04^#!nY2i?Ec-jF=48e`X}ip49U&H-cG%vNB@at2hC+@xT86ZDIZ0}00p|G_Cj z<9^k5;EpmD12>9$5xe3u6F+UN2wjY5`#Ph*|C+rxl$n{F_qa5q^NgBe7~PX(JsvVq ze2=H&nCZE;>N@(}B1tEGy~9OljeD7APzEXD6wbPdG$H;@duq>wh9S3I#;9og(r0%X zI=Xfbx`8r?AcyRufb;glMVD-vSS4%)%<5$b*uitz2hQ8 z^>c(HFXYffmoNNyDb()L?21K7dog|CvNK0rLG=0+pprHke^daY z_KECyHU=;IMuUWjwl^)iyFC84e9Q8B0m4Z)W$|R@0OhZT{3nKmJoJu9o+ns+6F4hx zBx}`&c`CgawOLDKxSy6JQ7h`wP+V6geAX&lxy!ExN|i-JDMOD_$ne>=;+{EWdzOz0 z9uNA0);os)^5D@_iRM>fEw#~FRt@sFn`t&Tbx$P+m&82cS6;*yxl5*XBYDj;pO)Mb z2C|HBNlv1VFxR!w{BUD9fdC|{b$;{8^;HPL3u%#|sZ_1hw&QBI`ec}IJk*7YN(EC8 zNA<2}_f;LAA6&Z-6qMwxeJ=QsK&?^pg-syMy(fxbCb{TXQ5WYDgM*VCHmc|@R|&FR z;QG+ztLrZi5@j7K_*hvvAF#^7eE!SL1Sv0lzyuA6iv78B zwH@;oAw4Kr6E4xvM-$7Vqt_shbaHv+6l8N@eP?0Ux?G)Tr^I5FZN`HWVrk%`Mu?8Q zV*rPO6KOMxm&c-E@Dg==m+@TFS~(t$gV%B8KnOX z@yto-#$o36ekj<#--P9Z6aM&WoJJ^p)|sj6iW9ws&b8zdGgZPQG zUtNhp8&pC{b+}AZqtl&+GyZFNw6#}WC9@OYq|G1Q>EQWj#lCV{rc#&Y%SbfUG-q)K zJ+SVB1Gmw7Tj4)IIknT%7o3K^?89k=d5~SRMWq%#oLwjy8BQjpPr7 zZn6Jk1tA{t$Z>nN+F2vO;{Ggv4k zAbOW73&gz!ALy)-%Lu1o*1xV?d=SoN`yE~vR_ncSJ8aXrX-tg{6_~)gATKm+M)%<~ z!#WqrYWQaLD8*UXSA{~8-v{znSA829xzU1s!C4Nig6`o(%gP*K7(31&g0*m380(K{ z{K-<+!S1)4`4Rj{i~KbhV_Cr2a-NY|ur993>cPXNYl@+hO&jY=l&9_c?tA&eyIav# z`X1UV|1_XZ6|P4*aJse>Q%ez-oL27k?()47b?+JdX8LU5%7FqUkPHqXjv5n6PeGqo zMZF(|L}4zcrEgCGDS&me?%199GieFi3a=OivnG3r=}e>e%>7FS7D-$(XP`@SEuRwG z6!%I8Dd~51BT0$j8^VUQ5ghp}F&y>kn!^40vX&<767QldZMA3nguk0)*NPYyif?ix zPE@9N)jn_;p7TU0KhtpxD68MF#xIZ3L>o8TO>XSOEVK4)K5Byar?f>#4a>ItJmeNvpyP39z zsu?PWNI&CDB-cw7yk&HPv9{HPuGFrbp^7Ua!${R_3zcRL}Eb5gf;d?^? zfjeK#7v0~Odq#d32~t`@-e{!-KaAJmcI;AqEnDN(8+?1OQnPwGdfepd_jiAmCi&=# zI`GLB$QjNtDR>hc#9J9PLYZws9G7Dvm$s+6(g%d{H(bsaISicg-nH5yYy>8+6c-w- z%a89~vK?EgroZ{ST@}7?#nhe)e-mNr+73@p2^xjX~ zw5O{L2pnbF{_}UKuQV!SvOjKU(gX6AN?sBJnix4urh6)`amJ-Tfab z#4s8mzqjh4_)!vm%ERAdFnV{tQEO0a-h)OkEteWZ2y!iwP*yhMHW}^pRc)>?n+p6g zDnE`G(5==@OmiJbS9-v7-#kEirjV&_*aM@>_Lzu*&H41mDlu89TZ=!pefa{+*-kUZ z+Vb1ZIG!E7H^&D>5?ZjAbH-A2O_@|BX+=bz$HpeMuyXVg@l2N8Ft#?sN`LE{(*)h> zdPxVkJFd=4eD6v91ZlQ$J~Q^u;~9i5MDXor`5f+rtL2dqiaw3Fwq186vc#73OJ2bN zNe4gA($dxzX|c`QbkZxpOa$I^jq#q~KVHy0jMvY!g0U=+&JQZF)Vw}6S7^msbZOOt zZ<}gy=M!p$(<9WeF3&b1vE)t}UvsqT-~9=>Jj-;0O6)ZqT;i;5>@bHe*XF>q$udj4 zTbhcw7=cQ_nYb%*W1*>~Ab<^deq^GVjZ>vxsbo4-Rr8APv6wcPE^PAm>oiOKci*T| zot5G2=OY+e&hmc7LIH{^@BZroT!e49GZ`a&8(jN3n>{>mnJB7z*fkl#=C zBO_d}mhf1fT#8)!a(4u>$wmkdPuU)RkpiAo+y{Bv^{RVgXRW~fyZGO4r2g;sVvWVK))##%mK*qb51x&Xq<=p_4SG~lGiDgB z3dk|3x3JXq87VC=Sq4<9mMh6UR=0_L48s#F$JAcS9cQ2}wbcBn;0gBk!d7DbuFG+t z3e+^dUrl_{RB+yX^cN7b@HI1f3_>sEnpsB;Fu`zTu^ZzutlvB0h*)aN_r`)=tJzO5ujBUfH5pE zHY}s_^;v$V?tIi7Z3B-QaQBaV<5Lwn{964-zB-qh<*L~)a!XGf&L-b3B}%e0u;vJ* z55Wnk;HN8s54(Om<8EJKi3bcpi%7gTP=`@Ic&2_w1jz&iig|%M<$>P!C{kYtwSTCU zM&@WPbA|NL6m&rEW0z}}k9+&oSNnrkXZB-Ha_2P*Fpj7w9`rM)e3n1G5V{343}xC_ z4_zwu$*n(4_`#EFaM7i)GIV6ZCur}=f^Dz4$No|cI@;eU%3L$9&re;P9gQE}d7rw` zdQChk9Hc61(SNS~z_<-Jj#gJP0G200xo%$3#F$6Pre=bKeP3_T1U$088h)Mgu#?9G zemr9*PKCr@Z2xWgfBo;mao-TfUum-9_qky{Z;1ob&y-Xa28L+3uuN7P#Q|S!GbUoD zqQXU(L4xg!Uatk47VvW=pJU>M+OIyWT9lV{nmg+Otpu9x)e3>`vAWC6bEr;P-;nO+ zizm9*0(D8mfW!6~S~7GE=yJ<+Wgosm!eK{mkP@$MTr`kF)38z8y$O^L{S`x^*y)+T zj^2AS_T;{Ht(^dd!gza}?J>Sroc4>n4v;F@M(<|?IDceCKoE3W3;tWy=o#u_U1j{r z6uD;75mZ_K`$hfNsYMCYWUYGil#}WoWwOdl6*k6coMMNZH_mB`^b{PS8n7s4Xo_G! z?r+&m(8f~KX-A$KnL|k{i>7x!i?~Ha^3nD)@S&~sqb*5O2vXHf`~7&szoixKLdTZd z;%3|Ch1qSoRnoYt+b*ySMs41w;bVhe7{43 zfBR~Gj?HY{h26Ik=~R4OzP3l~#dCon&J+D)nBak@ubDJy#iJ_hLjx%3f6&@Go{s7Irncbw`vYPh{N*FUs80O>Pe20y^4T`>w4{&5 zhha{5YV?sk9KIU=cZn5a@m~Oqu{2x6d{{pRUVx4JM_$^L@_t1S)(ej@VaA9?CSHZ^ z26ds#Yi70-T<5=}fPY6T`JkguoqsZ5_nCR@kOmc`19V4Q?gV|g4G3TDE8{P9-LC|ol6ycAs(vt!)UBN%*Vf~l zx&T!Ajmwrh4e$v8N_THxg`Ej7HJpR4HwsK3^Xa;Bx{8OePq4v^4IEeGN zXKeoQH@KrthY{q&ZOGuo!B+JIG5F90Ixl!@0vIOqD#yA8Uu+O@wNRLI{c>ojs-F8@7{G}$kAuUL4KmXs%GaA89;Adyvo z>v>vTJ7KDv<`c8uc0Bez)>zP~Jo};JP@Qvt0hj3hiU1F0E3v{b>@fVHdJ%GD9okQM zIRJ;Y)bzf~?3Gr=k6-e#&V;VeKFd}qB`wF^cN~A*rj~&`z$DbqD=yG;R`rYgIOxbx zR)P&0@f>vC6b~Jy2O7TYN)JNOyR(wh0sg~X_o40l6J`4JCPK?pO)ebNzXNoA!J~ao zIi@N*v%|VCrF07bcihdFMsTOjAop!h_Um0_pnzPYOV8MxTkuP5BKN@KxaSms`rFn@ zLz_R0R^VbxsYVp1=?I`_b%v~G<<#^JG}fT{Or+LPNXPqr`6r=vHrNi(4%q@~V1Yrf zopy(so80Dua-J=m3RFtFP@m3Q4e&34!k3g>-G8CB zAgoN*t6sm~!B^^k&vu{~0|=LFbOC--lx+#*wlSRI0*AmSvL&bB)KY^?TH#`mITcsW zk)^EYV`tckc!BOyDPt@Etld^GAgzZ`VtQLb+vYq+rJ#oCkcOj-T4#I3j#fQBG zB$%6la_6k5$|qD-;uHecN=**>KHv7=!}W2d8gq9jpwDWUbr1o5AwVwI8&KQYh5wG= zU)}oq^?YFM>joiF&}zbxiV>4lD7igA?%FALQ`yQ>z_J5f(FG!_TBmPpPi435sa;@E zcH9;@NW8ohp=lIGy%)=6*6QdEk=&`@`co!5+=ETaEmnO+35v z$XbFyRv0Gdyog~n0K8Qud>I`d+o5@g8(B}-7HEqh`l$5*?Z`2JDBPI}w%Em8edyM8 zKi7lIx+$9tn4Jfzk%+e`VC&Di4x1^S25shl+IFC7r+jp=ldZw1I?=q+VWt+}5yZ(M>TmRJq z@LxbrxT@^f^LQ@@Fl_%MN0$md+aF^~FgYg9`(F>e>~tgiP_l}Qj?ukd26~VQrA(IO zHd--V{b3s_-4cUYzRH6kvyG+qeJvcR+VMzW1eY!V*!Kjfx7quM{`(g zID_BUd1@fHLmne@Jh*5~h2F|}JQOGszoxdV%nYk#p{mO&MnV`EHnL*)5&%9@-~HjE zo3ntY9ok4cOxVKzZ9SeAsy>y>bs1~!sMDkgUTumL_iPIixV=r^Nw7eZvA{&_IO8#7D7h=sMj!oF{ z$+KO!$qBiz5OMZH3B#!opiTFgf;|cr@bFT=BiJ^WA)K9qV5@Ymp~9})&5p=_ZpV5p zN`nw&Ju~~+l7kEsOnWblnkI9d6-H+YjMv(!TT?QdJWcAFS1NJBrc3{>f4CpjKXNI* zIW)U8ffdhF$-o!;;?K3ooTzm$Ch@&K1EZWatI$y8{HNTQCh;#q`&1>@^6$7v|K_6RbB`lKp$Qi+{k|8__0ZcFxskGRZ+k;U zOne{HJj8*kypvHkJx17)nm!0vs7Z0*dp}C#+m(HlIwxfnu1WnTap>|FS$Mnj#(BX4=L-+Y)~~^fyG-8TVG} zPwX#0X}atB2&I=1s0;n}$r!Y8wIZP3QfA89&0^eSsg`(9Qb74g2?JkIT2?e+Hj`~* zjdl~EBz5NbtuB3DLB$GIs-Fyepx*W+=;KH|Q=QNHt$Na@_XUPw8PJURSfzCWv|7$P za0qJa8onKyaS-G`M%*G*es&%fM{$U9ZTBWP0R_kUR8lo*T)Q4`qWlK3eDGe-T-_zz z#j)RLt-ZIJGg>4Iq3FLs+V97uI!Hx{e6Cg+Kzwgr&e*?kDR9S!hj? z5C~1OJf(FkwruehxypHyaJoqV;Agfg7sAaJC^#`<) zE@;2DR|W4^=wx&$|D$PSC#tA@y*rQ;nMy?SREk;hc(JN{s%4!S7Y8Hc+QgRqY!nTr zD+j-%(uq$P#*<3E8x(4@#ReeZ{i7F*; znsRqqd~X`KKcPl)I1x1V0kdi5f#Ea(_nRY1#kTqfcw)YEJk!lt;J%4L7OU14+nP@F zxXskoSG6#Zy3Ki(_YX9eW?uGbw}2r2bg~aXT|CPsayak$eJyQ&mYU`n#cb45;z5vO zs(FbiOSGcqJaA_U506SfwaQg%qQ`daSL-f^EdY1OX%2jM^DkHObcNq!A7@@|Puf&D zw5Fbm?LHr0vZ~$WG!z7zm7>GS2CBF|Mu9LHw8Xq#b51isa5Ujqb05`oOrS&)qC&+R zVM0or@O62Nz4m-E)3yyPfor)9j+t#Ht*LBNGHnh>t_44Lr{tmEt@q#ipbv-|f9yG4 zU3q}NolaZ+Gt-_p?cwC}wch)=>)YY_bZ$Cs3FTTw@yP4nwb@ntdJ#X^1OFYe)|nCRPvNqZCCKZpW>DzC-! z{i>(7VS3Q-H24mwpUtWzn29Ve@hg7%BadwhpXV>2c+eH-?4|K)5-0;solcs4j+5tw z*6cmlnQ>C)&Q$l;uVzUISdLO5uEQ28HT~t}5Mb}e`U&3ky!GKHuquI{6`(VH1d-1i z%iEqm#dFOSasVKOXW8vDa^I(@MN-&Sz|;NLbcZr`j>mcuV*Gp1L59DQiGQE3zDLiw zMwuJFU|IVYiFjs81MGt7vN~}6Po1Nw6D0X-LGgY)G>_$U?p@{b_Xnai9~W0Q$3>}p zyLhaphclDJo zYhGghkzIj;A1g=J^*$(arkb6{xc!ogt(n61cyOj0f?k8L!U_!eh0c|kcVE|35udbB z@bpUhhL_MzrH(L1n!6)#bX!(*_;8c>O-g-c1$}Z^VWH?`0_*Z8ca`vol@z2^ZT9A(zLoKGV*+yY1fyCW7Rg#Q2=%XFE91mP8>-8d^UwM-ZIQO z2SgRz;?_rYCjQF1eD%)2?8O5~OaWp`;Vr%0NzP>#wHx{)_}yQMQ>PelUTo$Si>AEi zTz02DIHD1--kbT%=>cN}z5a@yZpWSf5v>H7Gye6U#9f(Ny(w^PLhf`Ii%#tdXNf1Q zIu76Rcv80fw2zz%$v0xk^b>3oD`)XJ zd=YaLR34uS8v-ual;Cte{curn(fHTtn`zWioNmtV%A@gJXf7BBRht|(k_`yrUlA2APea!qt~^{ZQ(tr_?j zXvHpK;4HFk^;D{lfFtuv?-!1IeR%d%(#s0HW9Q2JXtz{>$!yjiJxvJb&9(|pv!{i% zwH}w{#h>T1uUT$sI&7`fOw~DVIh$Q@MfKCP!Vc|fQkWSk-jzF!4?H1yH>2iWB-eUY zvB@&G{iwU(;;GMuUJ5z(=A9jPLJBE!gfVOBTTv8d9&q22%wp=EkYu^GuNh~ZasIpYv0osUv+W8`hIPc?{%3Elb}$V+0k2de>dXL)XDOz_Ru+P#F^B()U9!B z9sK;E{U3g+PA2F8`V|xpbD46_E(h&`4G8xKo#lUpCOwr>Wv>7E^Ft$DE2e){4`#IS_LsJ-Bo78=LATK#shHk;-%&y%Q4LYBaM z<0rC*f-e+0Lrd{+lY8E$8FDKT0`UFEsu}^3`Ds$F?l!5zt5;3pk{QG3f@%xSntxv+ z9Gi9$f;x_{kfnWmJ%WrWyU9W~iFF3VpDt-+d@7~KS4)M^B|JJ*(7_SZn)y9%aN;{wd5P7?6VDPQ02WDccRj0%k3VV` zxcIW>sc74nn(}753kVZ+*IRa`Sh#LBknzTaMI7Q-Ks*VC=Z-h<68PeH{f!&#T8p0Q z8kGJObwG6;KH6vdY)ig{!i1dz^Hp{Hg_j0iZ*7Am+zWjdxR=rG`yuWxuFaZ8Os~zK z#>MWIx~J^o>a-r}S0g4@ik!d~i#|G64z?GRy_YK}*FHSOR{Dj`X+6x?K%wPpO%xSw zrCRpY5lkN+8jj0Yug2IivfDMrPoJ?#_MwF}JM+2x41NqvU(D3ceCy2UIt4jl`y7uC{^y>zh5$VaF)Sns zR2muI z>d&7EbHTbEWAq_N$}v1bK^0=3&&QOPxnXCL!o$~lcUoN`ZG$kUru7iTX51+&xnK8X z;kz$ok&Z61x%-jW$R$d??>zRQ%x>q(s$Uoo;&O|Rdv>^7Gs&rd8*KtKbPsi-sN>-T95}oVvhxU8J zPhYra1IoQF=6)Q9S@8I|XYED&IsrIP~2y{r!{$%SVorD5v*K{x{fETvB~ z$84)~+pAuz3KNOqSb{d|8;;#E)?iGYj+*`DbKktk((Ins0$&MUE@wD>YDwK1Bzenh z1OC*jU%HU%RpyMVQi2Pd zF&(g&YcOqzuk<*{I;x>}WtRHYAUEy8*l~wEp?Z1^+?W--= z$EvN-BIm3W{f%1r-ci1==1$MlyWZ?-8TqCzs|t`N+G`l2+m*LI&2n{>50DG)7y5h>9S#^sKQBH`YFM8Trr|h9@J8m-PA|9>_H;ad19}FEA6snVJ3iLHR%Nx& z3=(LsyyYe^bCR^}qgUYUxc#iQu^dC~=l%e2CrXrEkLthqPQkV6kf|VssNvOLbk|B0 zTL$K2Ap#-Y6iM2^-;Ddt@;A_>CeY?ImFI9y`C3ELvCFKqobOy}lvXs-0~n1~36g0% zz^3}L>C!lVD`BY+Xe*zKb2SUEfc9%WUE9Br#?tJ`ihP#f!;$jXWeK6fmn6+XAYi|> z96Ai}o6w!*pgLFOO0!(=ujCWwnCy33uKo?k14uZB`x}H?= zykMxAW(EqBRo1uOL{-3^D$LI*>rk62imnKs+O!Xek-t)fvBa*!F&7o?d;4$IqSG}? zZGBW`jK;d~3b|Ru=s5M%yn*Jcxa;~OBZ%#lYRA-~m+PN|(>H;D*Du0?*`6=X7^Hgt z9!Mjdw`Ae+vSPT!HW-bTuw26rZ$?vGR>yNQn=?Cd=op>7`=6Wa z^TL(S-QA85tBap%NoA!t1D#x?GkEc_5OILD!I*sx$)-PLDE3; zeKs(WCtLH=-|3A@GX=lP)bifHABn4Jc=6brga5pOca7M^vASoHdqKB6f3Pxuv5aMo zwmfR+xQiGs>{y)l=%|{Qb+$4iiOP%v4W)lm`_np{8S=?T`VP#?l;sC-^PPrDoxX)h zKEfr0q7bTp1$8{)PoykTdWBpMS?haEz@PTD7>9-1OTz+P(|z4WI8}jwj zZW1JO7uE4*_Xt_VBQ8>A&MVQU9zRc~LuX8>b0#_@ExdHneri0@u3!z)w@}JZ$3M8| zMZtqRi${5elynkYbqz85Aw&*fIs-9hGP3Z4&U!{3o0b{qH1GYPvtnbRth({r<%yDW za+N>`%0Mm7-E zeiOy<1-c|X#;_Qe;3-Zqk$RriROCuEwN!f87}}*j&%5B`;&rHCuNn%vxqVd~%vOZxD_e>bXR#5iY=pU!NKo=>g8OS*J~W)!d{{<{ZA+va z5`}1M_lvn+>kyK7b@9&fgA9eLb zd28P0Saz($3~POVx1h23ODo{wGlKEgrBYUmZiZQ$vrD49UbT&h_+YS z4A0~(X5grz@rg#F?1H+kL!RS98K5qHR48|DK~Rd4Jz1uvsv|i0OY^Tf)}Yzu`{^& z`Jn4ECyNm;^moz)p04vQ$(&4MK^;dYx<9oHQ#!49&vzK!Mx`R#CP_W;Hs@l^Qq%hD z8U#-?>yJ0uY!-TmBIyPWhyI?bL0_M{f(o*mjAx|M+9U#yb?C|6GhHn7Kc8Hz!|Xpz z)L3N%A>!vSYM5?zeMaFH#(Lf7W!EN+7BjWDxJ*Kh^ovYvCvl?$bQ ziepS`ORTFqPlkO>3!K6AO|mCP`SwodeH@?WGv)*I0h$`Ia<(E9rFdawa#GZs|6Z4U zv05(@m(E$P1Q~0pR}-$nEW82Lgj<|Ln8v^23btC6Y(~U;ol6+EF|<)Z3v#|RzvqRD zT}k21TS*aqE)LTey~FqR%}2^mR@dl3?R4Ig6oy7#ukk4tuXeLyr4I7@VRg~l%ergn z0(#3S*9#H2Yd2(M!0 zWp-!g_GTyjyNQ6FA!Wpc%L+|hy`1dk%yg|o0UBOEYcZ<@<<3vduEa1+K<#wb43^W% zh}PTQr|ItEQr=?ZgkTTKHS(#KtX~yi7-uT~BN_0YifEnlQWhKv-C|+;)fjchf3a&( zC%h{ivwf&EGW}mm?^qh`F-3{J*r`Zb>q!vznagCPi|OpYm*)>5uh9Riz;Usloc~4= z8WV0+`h5-I!3En0O)SVhoYZ{n{yHwbO;O-xpXpHg{w5l3C^t$=RXd8DVGEA1YOn} zMZ$;sqbBmV^MnBSH7j)A=@>f$11LE@%%YQ^qmkBRrTXDG`;YPsY3 zf@aK{l`OItj6qvFvlY1$O9ZCr=7#2+6uhwae#rh=tE_~-cp(H(5$U_iwAq;T9spP1 zRRFE$r@GnSCE%Y%T?|}PY4F}M1mCBsyS=Tdt2lSCvHgf)RhKsK!#o?n1`=b$_YR(A zaAScH=qq$$ovJ8$=l0gVr<%T`L0|fFjz{PB+ zr3A;wP%DRueX)7PEtNs&EQYLna1fipWY9f))5piLj>fV&;_ink02!?YkS+ghp$xC> z5oRVpn$nBRLe*MN)gAFBrBX*Q5sd0aIJDW(?roTg^7h>GNma(aX&H}U7oN5i)h6Nu z>5+9RlYpk-`z{wX;=1i4JN^^m$_G&+P6^2uCSbN77$7wxf!}}z>o_36^MwGJF;l zjw+==U_GCf}3-R&s0m|SnZaQ9y8{4 zt~03_58QAlxc8vQe*M0A8zPqbON0Y_svyea&>>+s*cAd_5(g{lrx&az-!{sus&4>x z0eU<$o3fJhuPJ`=@>G8PYx0qm>=f)MVr4|U1HeZU8Ms}=YZaT`EGgb)9uhVw$8X>_ z07h5FwKf@ev5Y_Snp6iqO9k)r03BGs7{?wSS)#*BGVx5t^FGumA;nDUiaT9Z0QXAf zsV8h2n=a)$Vf+k~#QyNn#O*(Dbjn%7B3?@$);6MBuX25z`;c?R>bh709-xrOoXZj! z(H+u`a~iK3sFk)XHz~7m>XB?0_?aqM-GkqL?5(ZH|7_(MB6fGJR$)0nr8#!WX69#m z5p!LXJ5Ib^6b)dC+v(Xm%i~d!~#iVa>aYk_&1;^5mmS!uRD$&u>A@^iQBvM z12(k)I4~jmT@3OQJB?H2>Qk^K&)2(?*_q$-^^9WPd%w~H2^mU%!@5niO&Wv*C5M(B?Vbq;QCfk&*tmQS1f$t^MLL(gCGN zR#dT300KAI-e(!0h)XPE^=yI;=eq#(G!K2zqbIKbS)#-#fh#{t%++4E0Mogw=N+F3 zmu=H=rTUNGy-H_H1`Pj2@&6tXSGosIjK?JX0&ItmYBtZ}1Vx$_EcJ*x*MTxG7%S-m zszH!q}Q~RZOa_Rn*jVBaKCx-@ipd%qSJtlFT(OV$}rBHYp}OdW(3ze@yjB z*0X);63?~nzzs}MjF07;T|Sb4Img|EKunxEEOt0(;I&iW8;szZbFKfzAX6{Jx!ib? z-nnbH2SHX;0`sk-3G$lf`1!H>2t`sBt7m5W)z2I#0cQO5Shm%T3&k!hPx?x&c z{%p}yQfm&Y=1X%+J&miN#?WJjJpCYZx%WhHr!tiktP)$3-2RrbxrtG*Aok5Qed~X{fYDXI*LwDAWENDQF7&$cJJTo zD%(S;*2id#L!$g$Mf=MnLcO;0_?>Pm0Q1}P)_ngdqeR7P@NsU1oWAYj|h zT?AhMae5waA0oOJNR_ioX7c6kvk11cp){0}I>e)5PqFV%#cC?|L{P%~!ZQHgo;Wi= zU$1C0rnbC@;Jx9*Lou=rsqO-#b}b=o2^Rk3NouPP%HmJ5N_@c1T(&zJV7cA444c@gpwguYwi|N-` z#Z8yOB&RmKT>aI!+W2W3FAUL5oPqCDJ^LkooE0Nn4BARwNCOsqvp|}hXUn7ftzqF(y;D-~Q>Nm5<5*!^Z1>p?r<#5Kp#%204-?AACMvq*Wc8SNP)-wjEpi^78D%kYzrYw* z0ac4LR;+!p>IxASM{@F=)9q=9JUK5zTjKr_Rw4n@BY`LTI^}^Eo!J&;y5jqRQj8^! z_dj=bFg_UB?R}Tl#0r=u2EPM;-1t`LJkg!Ri!`=gqg;@YcU$!N$>zVM#wH)&f)b~a|4ga}d%vbKo1_T5K{H0CQjrN)$#bt@IR{Rp4yexh&#tpDprpbT&m9uUPK8&f z)i^aWOdTHpu8p2a<`!OKCOCfBhDY(eTI}_`X}0JG?sUSH(rLd(pa?WVo5OYPiLtir zIW2c>XHwlH~HEE*OV04g7He87X~$lF?=N7@GjKfyItWtOoE)X|>XVa09Isp7(Sx@#2RLukC0*y=V<>#} z?U_1*X|3WfRm>I;UM`5Ir-cB6xr!sVZiAhN*D0vfM}OGJEnY4)xzAvdtrs|&CscCB zOl-rZ?}O5Lzr%P_;|To=4Gj^Ed8QW^!9iJ9U!sCzIsCEVUr;FY8>C6g%@-v%Ar}_U zMcy;u7d{_Fy#Fn({67QlfXDs|wgW=bj8C~F6^C;N0?$c$k;sRTmImlR>zs8WO52JJ|v5WiPvriCUrrKH7tXqgAa9|H+fLMzS0nP+D+K*JQ=wl0MA%n*`$tHeQH{rr0ZGpDOXJrA@+bzRdU33*+9nlt z&Aq2^ys4$@NmJw`iSiQ>U>L1UnU54OWu`8V9L)iev3vd4e|;>Qv@TKDGv0-PEkkF3 zbd{MJdcIE?TbXFaJ*DHF{wc!Yl^EJ<*K0R-dOfAGYykwsx)Zt7^$!>ybFOQ62JS%X zi%*S+2xt<^Bax9t<&UjD1#O!R%D9Rh3Rep6vjNNCtoYZjb_7E-$NYFi-R@4LPrVj1 zH~v|fp%m1YXY_RxTJmV>?2zno?dNnB%)5er7fNl{zxEjSl24Pf_aOX&t>-k(`A<@t->O67*P%_>(v0YNw|t8F)5-lF{wJVP>Uk z{|gZ=2YK}Bk4oxF+5eYHYR?9XzUGD++kIQUjeolBQ*m8UE|JsGe`euy4cMG4?7bbw z9-arr9$n4vuXXuC6r??*#^A*QEW!$H6D81~OEJ4PDyV26hukd%oV#H}j?>XdPlk8V z>?A{1{aDFFMPs*|(*Y;dG=4k$7K$TAHyycM<0S(Ib=a1KbOLH3Y_G$Gvz);b$SRR4 zo2PCj=xg#dOm^b^7vx~MD)2WYsTE5fNAo{Avptss^fH5bUa-zgRt5J zw@$m1#)NgTa_y!fk;g29>WQ^HKF2loC?j9*`ZTeXx2`$uFWGJTz5n)A@!wQkS2>^* zhO<%p?T)jo){}`CUUVRkDS0e$n5*Zr&OPIejPrdi=#uL(YO7BV8os)kiWkb zS8;;6rbWK%2?W=FNAplBK8VO5plIvGai>=uSqc5F57(@4(yM7SEU}YQY_F9=tv?Rs zw|=XyS*)-Swq?Tz0|T8k`mFbZ_TbjyRW_5KY%{)mH85xAeUP=!V&mhoFf2a!|P(BU#Cw{U(&H2ojij5h4B)+dM!>t7u+t63|jv4!o}P z((um2nvP$yZ+8|@;5V}y9q4Q+3l`1}%mUu^N$rr5hC9b+UwS18qj-kvEN%77~XJ!)D_>8Eg$IiUDyVgR; zly=5{v#SOYh1_wbnq0JuO-S)<3bjSG4Mp+`kX6x(D%ESDo;`9L84A8bYD^p7UHvJt zYAYwr+<%Ros_)(p@mXLO;Nk2d!=o{MYAatK);G4nXCgliu7 zVh0Q~U#F1VqAurKwrtZ~>)HUjp;9G)p)#TiU9&F}ATNzSI@Sg+YF39H)0rUWA&>!| z;Z%q<5RrBVJCO%=&1G#RJ+8R6vpqS_BShnVJC_xi#+(gkjH0L|w0hFm+_{+^1;P9a z+c5sp?>Y#L$ETw`(?HA=wBp4Qoclvs2G8L{-RrsY@xYvNsR zpaj_3CM_DZ-Kfj3y7>y{!3xY37zsvMn6nTbTzqou1>2CSmXfMXOzw~OblFWEW!&`e zm@xVReciYs6(>cDevX8xS5rF>1x4wZuuejnrUjJZ#w#bRM`oXw^)MUGL|07;&`P6_ z@U&b=TX3{hdJ165RI8c1*`|!+#u60y#ilX54)2`9E#FK5*C4rGi1XI|=5PT=lZ*dQ zCo|5YQh>6zoo^Y$E0!h;e!2sHTBrE+IrrS;5QOR1_v{!Y&w2-j?fNA#1vS-eUQrkV&F z<0QHVQ8r9Fe*@i!$Op-}ax$~WI0_e09+6pA=e9}MT^8>W>bOg^b8VBZ@czADCcCgw z*;Be9KxmFdpxNwfLo${)RbFq8`lw{#hxXo-mV5-_v~y7Rn@=&n8S+Mc0S~KE#FVWj zMR%3Z<36XBytF<^v%52+&|lUx>Ud|I{0Lld1SxC5Jc)2^?CRhu9?cQggm#6@Khs_A zGCshtJcy=;*W)UXG5${=9LS62T7&S<8pEwR+1j7HiklkeuPn%`njORB6rqkdo1fjS z`J;5LJN32-2mIr6F99?8DJ_U_D(J#veN%aZ*A)s3lO9lEIEwK#8}^;19M8-p$_ixcCZp~+;tl2 zt)Lwt%+SjdBv zM3{6AVx|ir&agkwG@xaCA{-Gc$jNAmT&ao5INgc$O_A>g`PqvJ@jARu#$D z#}|qs(0c7M#+yO-FNAnkoidORb}c=5w>!5!Na$gydfoc%%rXPpYya5;Ix0Nsm!iI^ z+(yso13_ea}fv8){3d5^py#q{Byqqb5)G}$s#?Vfc$2!Yr%rc)_ zy|fFQZ96%!P~|L`{pkr(wuL)sK{h$PO`BUjP3hh^ev>I4-&J$G5KmBkd?(w7PvDvq zem56P*O>=wDz%E>bHaF+c`@(Y0R*QSk)H#?yaIZCp4;%g&?9%dY0UGOHq7s+%KBe5Z|S464jJuR;oj{H)Xf=@$?5x7D@fDGCLbsL*bP-BVlyw5vAL$p`?5*$<3n5vdJu40twq1N7UVCtG>@iWC%C}kv>RwrbZhw=^k z*-lVKO0m6e6YNoLSir<~*1-T)ra9U>w&F*9UJbvw>buyY22Lam1+xOVN-=RN`XVLF zERzr!dBi-9`*{D^+aot0HX{N>we8NiXf1BeCD_IjLU8YT2{@f)i3(v6@)Qkf6-zIq zc`!)ZX`nxrQCt%8b8Qc%R0OY*kSt?xzRc^}bC4CEt$d(AXw*T=ewGm}_iOF*e|U=W ztUdz1{IViSQNPu{!s_hnzM4ca9|w|TX4g)#@KNha{>ip=d%6U4TwoYgl?gsBpEc&6 z-`#LRe??V4F2N|g;uL;s5N~U*XwOpgzGQZ01ex@4aJQQMgV`8-{Yt-BU)QeXuVMYv z`|%%|!a6r4GViy&IjjIV`QeR^26@lM*{+ZYS51-Im!?5YnF*3KPFWaHrJF*Rm%S5BvRvuB$SiZ=F;UkEpt(gl>|1w`kN1P@XXT)nvmQ^Jz z?BlFT+q~hmH=vuM-Pe)#Q-`yrPrG1Vx=OXJ_gYfie*+lxA7D=X4~7U;#LZQfiXY@7 zrGOoEmh$r7rScUWr?d1rdMUyqNV@lhByU}qm0CEGUh_}HwY6EDw^tY1k>1lVSSo(C z^hX2g7eVhUw>4yrd^Gmny_Z_SZ&mboE1dnmO1*R<{6vKF>idN3h+yB7Zy&0502HCn&-IfLL%w1t|*vpZ-#UJO%*&rt`M40h%(R Sx|<8e0QT?obSiN?O8E!zq~jO> literal 0 HcmV?d00001 diff --git a/opencloud/charts/loki/scenarios/images/removed.png b/opencloud/charts/loki/scenarios/images/removed.png new file mode 100644 index 0000000000000000000000000000000000000000..219d64c32c983e72efdc914e7ef31343dba85fe5 GIT binary patch literal 71762 zcmd432{hFI|36xvN-7gY*$G9mjeQ?k(qa%QWNYlQWf}WYVGu(iMz&CvY$4m&Vlei7 zXH52NV>iZN?ufqk_xa!7J^%kb_nv$2IgWG8yu9D9_v^Vl9?!?~ITQL=<38Xb)5TM# zP5~Z1P|-eh>J0eQseckFDafyk^~>;|I`#V0Llwm*9>&Yn3yy~9B%D9CJ%K>|)-#&@ zD%JZJuY@J(AH2TtK=A!T=K>y4eePG!x;*5f8+N2hKT4Hy@s!%NC0uX&P@-pi)3(^$ ze$dnwdvAsK$o|Vs?W>YebPuCG6oU_Tc^AiW^No2w-u!kT_wzg^bCDmqn25`t_sRK2 z3a(Ut?k}Seb@SneKR-yPuUkwcE{*c1k36p)zeN6>5o4?jJ6ZAJ%RB8y1stfq|Dk=4 z_v_Gm$ltZuX)(RF&A`8lpJ+W=5I0BsUHt7f`SK#*zl)#7XnO$|x&JP{^N6<^Miu>c z@wKo2cW(=t94)fQ@%Eqa!S@u+It@Y>X0ml~q={`-{3>bv(hvr-(zk;fn~xNBpLlC+ zf3lB91u@3yoK$^;P5N<7(n<{rjz~Db4Gc9^Jk8OqNO)N7wcCR&*Yuxr!7X8t ze4VYmXGq(xG!9^LyE*swz0A;XJTai-se#WZAD60C@5v)ASn_=x%A!Q6JV7zd1bx=R zj*Kum?uF`Bz!v-u3xwwSZ72y@d&9!`0n#P_FXT>YfI^`JvA-v@@azux-=$DnOK3Lt zusEOJjN6O0I5m^Q=iIA>Xn%V0rH1YTXqVJ|-Mucm%yqWq60xKMY+dZ%6YtG?`lvw5 z31&0s?NYSCaGzFH@o;}$!D=JlxO&lTb!3UVmQc#PGybZ~j&O0PyPLsoWqf^_XZ#0a zv0R*xiJL*0y?Z)39EvjW8Obv$0~NEGy#Z2m0sLD84qoX`2)>RJgzh@LbTu#|&hkC8 zV2#APth=*1w%n}aH1>r0!fT@g*r}^nzk{_CO?&suSWOXu- zaVCtjZ4XB80{WDxsY8p48cM#qd=D`eMya?zYTalr?y8i;CQqSdefQVV>o03Sy7|wi zn`%IHfP({3ht__Z!_gBfYUh%zI0c~H>u^kIpXs}MUu=%FE{pB@5}d1FDOkKxMYtro zPyqLpx57MhtXhCSN<3^k&*tpEKg}*mly=&9Qs-dLz!TV0C>9RRLfwzB+?YrL^<_fl zrhHGMrCmw|ZHWqUn|7UE?xf#rCK?l&BH*y=LLJN4*2m4jprxkS#f1gQ#O?XcYgNw} zFx_X>j-8$yJ6h|o(54^IwqnY@l*26^_pWME)Cer(68B6cwpKQ}l=Sm-t*6SG`>zfuc&t6D)HvM*dz&5rf?Iw7g5;dy0;Y%MlxPOj_E(Q zKHJ_d?0N1Y{^6Gv*X7|tu#6Ga6}#AY!lSXh0U^yl1U%S}qqdB& zjaL3V;xcDBSYrLzh!&iEp|4GBHauy{>(NOvA!i|SXgr~0nky;EJlGQ}67X&5jU3+t zFpKKRkZuJkq3D=sIUmP57%e`QvV6JUAnEfdUV>AxFt_rC>e;?VJ=U@0+jK7-99q6y z5l~9KLe<8eeC7%>=gU&Lt?W#cvUgClG@VW=E|6mS{bOkuFF!5xYOK7NlZ0wGZ<_#o zvx&0F=<=;&=W8M7z$afIX;;|X+^4G#LhrN=eHCO!#B~FEu65F>W+PndR3v58BvIxT z7r1V6BYQ^n0B~R0yFkHU3r;>t9jhfh$;!3;i;7|w%^+>y7@9DVDBXNW{e#X#aoOO} z(LTRwXDV8=4^u^R0WDHAQRP1KYWnq*sV9a@J@R}i9VyrXmNZQt?}}`VuWw-vvF%W6ja))s;7E~Yb*>)eSQnt zmhzplu`8K|s2Ef7H7m}kW<4gL^4$=+NKp~xWN42l|Ki^aew%CqYZTolCUBd{;Rbz5 zL{td#HaGYx+$CXDCJ5rSK6@ozj4C|TEFN;9W3}6{0-IPKoKkH_LzY63g!oI5) z4Nv(!1ju}9C2}c43R~t_t8ibfb^sPi$|(|SQVo4LW3ZtiQ}FQM~DJk(3rAv_R66HQp>6SpRW1&Y z2^a`JJY5?WeQB}vBOk6Wg^*}a{z@xxb{nm5*fURR!zi>oGTECHNtqZKSzuyf8vizs z;~U@)w4Tq$)Yy^1AU(1E+i1}MtDMJe4UM$#`R_ufi`g=GzvgFx-eo6xFUL{KJsFh7 z-Z_|zw{pQv8sr%m@yr@#Cfs5xdX-P+Po$MNEMh;Fm_k|#>v{e$@4xJ@d>YW#j<<|c z=KyToN5(*uGBs12)H^!#ZMx)$J{hNxZWyoHNKvb$qQMFyBwN6m|LR@@^Im_Slk?Re z=u@zmpr;5H(?@HCd4IB-+`PPwBq`538a4KNy>Yd3O%crOJXmeaq73i642Hi2h?&@n zBm2i4wd^Vid1BNdnOn>-Pvz(qIMEA=na|FO zf{l!f9Pr82xb}EaKY}&1L{dGqX3xK7(%9EN(Zp&a!I9`o^Khih(SOiOC8^YIWRc7U z7QDRj&1ZYmFGdgeEqpL2^GjbXQlVFDSdY!qTi97q_|@iU>f0FvV?aNzcnObu71U}) zBc(ZLm7?N}ncXANy0WAU5B17bIecJi6?mv=tZ18VA+!}huE=N$MgxHUML>Kn!X zPAjW;87BFO%$F>BN9fT!>^(sd7Ofr|{k-grZMqb$bM5+NK8<%wGOfENe5<#6e`Du! zsQY`Jd)!OD>_1zbZN>?wMH-5Ba^Y}j&!8)^m4hm}@nohdSsS|NKE^3z2Sp1nOVlb4j z>;xcmPJZGb$@ddk8{G;SJMl*RM^H1A{>1nUW*v5+ny?eY5=|%KSXb{6&EKfdEB3=C zE0E~%dy*p_rfY|=ULHS4_Z}arh3aP1?L}eQY})ndU%Ps z53a5bvhN)C^oyquZ|fJTuPU-5PKs9vsbR{-Z4=LHDR21g_xW$CzQGT^R~5^UMq~GP z9@er9J*?@PLq#5Pw|&O^hQ8gM$Xs`vVk_MQmTT;I5_hK~59Wp1GutiOufRAu=Ve3i zK_>=1GCR};JuSX(e5nynj}=V?+Smn53ho@d20r~VBF*TUr8k-R@B>x|{uF!}&K&kxkIQ1IbfqS>_{HRTzjzVa(>Lso z$Ssp5_6q!Itc@YWO$t541KzNSxztGUhC&2w8t;2m`i25aAe(zjde5VPa8*{g*kVgN zzh-X~T)jwDWWLKuN$1+&HO)$CN!#HOLR6CuQ3-Y-`sdD!3+sbQ*KAaQvU1EU^>v|j zxB*yj==~2Rn9am~P`rIPPsmm0d^_EUa`fhZUwhMi}+F;T9x zLvXRQJGsjGBbMX4p&)jy2cUu(m&oFF`S9{X{xNr|ZJ@s*cAnuPV}>?bV0*+A%(PAK zcfOE?$sqb(W&Lk{RWvSC8%u48$nxfVs`6}wgF-c2<`=QPy(+2rk4h0k)+Kly8GaB(Jg2uOhCwo98RTCETnV9%k< zAWfe<+n5fPdyKi0LAm_V6CR?`szY+U5)Pxwae0MIOfun=#Hp&sb~GEI+cvK8rl=pJ zyI7sgwTCikJz~yV45Y&Y&&>lF%YQa_bNL2S74M;1nXsuB8M#|cizwuq=pnzYyMsAJ znq-2gXy;CeUVUd)bQ$oNsaniWxgO^QCa6>w zX)b9rBAqI(4B=W*YrFV$YQAd>cjm_rU~SDzx)!E_S}Pn^*r+PsaZ=H3n_?EN_g$*( ztG{b37e90_e2j3aocNkgo8~ZOYhjVe77`OEf`Z#A#Vv?h&F0ej)tEtIq&0td z%lY!}z20QZaE>roEDoAXrJ8NOZ38<$9RkeZPY~|NTqD+%rfg3nf@^#;lmb-6`b`pS zqriXk5Z;g4UhsFq;NOSm{b&7s;FtCQ8STK`__>aOPXa8) z{rjC2`QU*jUwX1GaKQ6*Lk=1+ZOznN3S#{aiz(kigWqlMMr#saqVE9z0B(XAEYj*; z8hI!M&@pq{elK(|^z*LQL^UI?6|!wR^qkc()V)M6(Ah6OCerZ(&c|zxsCVK4Qz|kM zQz4z+X(Fk)Y@3YSV-!=H|1kZ$&KN4}6xj&yq+^QnfrY}{Znv_kvfk!_Hv83!K>m5` zQ;W#xK+FzJ8)sB9tyb_viMuh<2W2sHPk#5M<@hYvA@|V*>r6dbef^y^BfVeO~qTZI*4v6xO>zov5@z0+o*R zk&hQyGZNfP=tMQg3kC&*4h=u<`F;1M+_Wq@ePBS$IoNqw0&AP>mD6 zfuMYQfumJxm#f-*;l4iLOdnaM*>g?ag27st!X~At=$N>fH_ZUrF{Y4=`PMBTBLZg=9nAbF0+z50b@?pIMie_^uJuhS4Gmh@ zQ04mM6ZU|7sMkfntXMFcrmCtrX1Sp}NbUJAjc3t0yj@ZiAjFS>bcvfKw~!HR??|pGt)=B`%$Gl6@#bli>V^? z>e_=CxtTu6-73u_Le7Yafh{BIDXwa+5}BwE;L2eRK}$|{mX$?6%1_JDpy(vR?O1|V z_{1nJf!06cbJaRmQ#x@U?$ZlL53rg^D)rngfr%Pn>mOe7VA3>-j}KXTxA*ll!y$F# z04w~*k}z0;_cb&$jYtnr>LsU+&glH-5M3T7Pl6?98&9W@psxLjO-hrLnAl^KIV~s< zYG(75ine~229suIIZCrsoOHpZQJk_1!!0;D-d5pwiIKZu_;zPPllyp{KrStgIl>In zK!b+Jv2maMTCU*~ATI{q@idzXe;l4&)J0FNHRa&z|>3k!#4W+EQW8tLzv zt^)v6rvY~dix=FdDc0^%(E>D?Y~<|cF9_R+O_0;CUYwb5o*486;IJF0U;SN^sVOGN z`aE0?c@y#g@evMX=#Q|gKgX0RhO>KnKH)J}K41GOf|yc#HdTNPRNJmM*c2A(JKE&H>s+jtU-NuZ zj1yQYQc@P$@A?`JL6RL9LDOru41nKf(g>k2*gcCiSLL>&>-i}rdPeL~y%QyW@U%C1 z=dRr$jEfJ7Z7T8#Uq+ph&HO%=0L?BWMNMUk!#NF7RG9!X-M}l-BGV99h<>m2njU8T z-GxHc8=7f(iZ+iWB?Me+kZE-eO--#1eR86ktTuzq#W?fHn=FNp4V&uO&wkeCz1GFH zFJZj-)PXIN$mh}_Oo4<(bATe%Yx_vgi@Vb=Z;# z!k)V=0j?Sa!`-Mr^n-bk4@A*Fi@ApZG7*98yOxlGgkSm7G|@ z(<piKU2rUmttR5+!8M(vyg+$ThahjR&n~0!`CurdrrrwF;-_h;~7ys%RxBy`vfF z<5TNI9dfEWs#=gnf4V;oMIolU6xM5EhtG2K+_UPw7(CTg$&|X>!+&_Dxk*UtJ}=cs z(v36`mGTggKnk9S-w#s2S2*e)&~|X%`J<@1e>&?dNMwN;+H$LZ_~X-X|nrm?*{7>d|xp)uko*mYl9^-7B)@9D#z zr15{HLj`{)0`@;_GZ|+mZCSa`PAgO-=lIhpQE~**C{Za{7xXz~9{7nKNxtEbz3CKE zqRY`cM{fr(kFIAQGyV_ySO2^;)p@JLhI}eG50nanQ2wjqr432O}bnErw z%a7}K=WBjFF8vmDyMJs`V7v_GTI#Avh%X#RUg>4pK$Ik(H?{&g~U!kq!Fd1>Pt z|HBPh#c!9rb8d@%L^i_*$g#E3>TeHyrZe4b9>PSDrYAqNoBZIn8uN)UNPhBYR{T9x z^zqqWQ>~!5#g4eqPLq*Mm+Stkc|Fm$;lE#GiD(s8npKVe@AmqEnlZz!1$g0)1+^eC zwvRVBP%#33O?tHPcP3>+3=psO-DaSL+J8=-{G6t9&A`o@sD2^F4BK3z+LnW@Wa37V z>&Qh(r;Rb`1=a2K=v$l9)5q6#DkeYHu$>2jOZ0&Vc;o{F+0VyQd-GJZ2Kks$ApeAkeB{KI68>Am+y!^RA3J#d5B8m9ig5-+Ls4OU zZD0R8!<`WZ=Q3x;sa4EAJc^coouN76(Z5eh6!S3vIfF6%Did1%;W&DBzv$#EzVSe= z`NV9dfE)kYo&C>U=ApIqODE_&v;0aN(s?7S$9L9jXn1^45O3Jg5?0agIYOIBZRzmE z0Y_I3%UfYR6z{Aw9*!Gec7@_+a$UGy3EXx`WOEViT#5c?X!#j-Gbzs7u&=Q+aVz26 zynEq_wq-ottxKYG9O+C&0cx2SYO{e#y(sde+)T7;S!chvkUS1PGzW*X%V z>T_`0Eq%_t$`Nd`v&rnjoh3g?ta`1>?Z=UQ;$=ITm^uecm ztuV1_tflH++qe)L!AN7T!+f*U=+LWiK{9L?G$o0`jl`kB?fSJJ2scwIX#%4D9le~^ zdYcyhS$ly7g_hCYzYDFL2V2f(EFN-b61M62jd9lnjTYd25%OQv@7`K3z>_S0X1uro zTS$Mpgn@lqkzU|H8T$!ja)f(MF%U05ozL~pDNjZUmls%e7$im~e^4{R)P%cS7wl-^ zYpl2g$8}_{R+R2^CRQ=eZjHE<4FECiE^3W$Csvu6zAa>$L}`A^s{Fp0se!L&wGXgBv&>H|?B_&SS}q1Utb0%Q2ymCk;X z=0~;TF1I#oiig6V+K;V(cC9~3YC~qerB;Zy0bTKi+ij4ct7%*YZn$mD37e1?ty7F@ z8vTjTwjsM$VR3outEj4-i4|+G&)VdjT3^I@*RZitTP#5StDxn@EbKk@JhvrpfadUS zIDmFrN#&NDBvZ#1#g%v$X`+NiQvN3KVppWg^Gc^2O@C4VwJ$4fJi)$GP^`3kfVs5% z7T+K;I+M5rr<|NPbKlqX&LCmCC6h=^lpX4xiB!bUzf+<6mSC2tK_o^j z)f0m7=bDlZNb|~?yQ-9{9Ci$G$ov>5{Gl>v?;&|@k(LA##xN3?6L2h zDaY;2*Mze>(8)^vs*&>Dsj5{~)7`?-QsB?x>9#w!Ljm*_WqXAp-`UW!GEMVp@(mVe z7YSJQ{k#O>`GDLl?V;*^UzfGt7fAZQM1T5roVkhQE^6Nnk8G@1W33m#8a#6h_QKPy zjzx5ppqHKegN*DQXON#DQydpkS_EWw=C-chcEk1a48?kX=-Ml!Es;k5O1#aCwHY!) zL<}Xar<6Tik{~Xz&u=n*?j(Sw4liy>(YH3N>*kIiHWPG0+Qr`?Tb+_ZTQ_TlE8_M9 zsd(LRN+AYlGE~>BFlJ&e-#$MZ+P4{Xj8sSpz;&jt``vG=Go#vw+S9hy&ENfMC;+e0 zsR)=&zohg2o99V;<9ueKgmoP)ChGOW-9k<{&a#x9sXSJ}x3eA`&+fl>i35WE{t+(^(Lu+lZ9JMB`4zuZk%?2ektwOZ&Gat8?ol@40GMyy8E1CuI0X7U$#^4aF(+qY+*?8INo{%Ok&!Obf*ZBjp{GTP z(@1%3i?{b4QGY><${y<)JhNZ#{Q<@UwZ*$yU$^-@R#J+5^F8M#{M&awj)(_zBH);K z7tTmqhuMa}prvbXVfj$@qUm6$?(+C+*G{Kq;)d+fuZRB-IU0yZuFJ5)=O&=PlQL6yC8A-!t<7C{~RGe7-B<%u8H8&Khc!bek z^a=XzH(;T`5_VQY%u0RAlBydO6ESb_lK~V&mJNBug9iLVjx2@z`yTAyO?D#nttyWF zA@b?(@gSPASTcVzQdZlU-*i)yzO9*<9C^%2bj~XH?0;TA(?qQ}T+T@OAFi*o_ zuwV%HI6ul9U`(9++Rk4~ocO2K4i=i3QlQ~6w#pU05lF|3Qc(^$K#W}A?Z4HP+)2&8 zjr5Py^0aCi2Xh6g{sTBO;o=C?XkbDbQj3Dc);!*}>FDmDqbJL$2#YZ9n_{|hPp$md zb!L@}HptelT+LNP5oVcbY8^J+YK`qKT9*f7v@23J08?F(ooFgrXxlt0Hni!3H5ww3cYlk_z}?Sr?dPq%i+ybVhsoBcVUmenG(|s-jOIj~}6h zvB?rV#eyl9$ORYU=WYgWWQVg4@z4}2_hae+Zmdfbyye_jbdQuL_$W!%2#E3?XHywb zrpgzci?FB^oMqz1K}<@Gr&~m@jFAV2_avduYC|XeC|9}i6Ih9^7|&XHEz!0}F4e{t z;a7-d-s9l+1NRDDgtm>FZ$cf2H@*=0Ag!5mtY>?jUaY%`Wbq63PcpWO8FjH&TN*wWV~2&y7QbFT^wMJ zUvE*NAST+kSO7yky-hB@OCi3Ce8%kWjC3~$mqQA8^)O@Z+3~WK)Okxr*)Ew873Zby zZ?sF9mIHaJs4F!$F$*Bah%$TEqoOIrxxq8&!7+(7S`odt57dj3HYU_f1%KQs z7uLRPhl(r`hgqLpOJ~bR2~Q+X;A12tz=KaYQy{HF^GR6`tecgrXNp)Yel6IC$PcLB z1L?dDo!NybJZDn4-XelBkG6bFTKBk;{vJ}KdPQ?^TJE+vB3DEBtP`gD*Vm{QaMe~j zGGB2s6eW}WGw*DfB(HLqV1UN$OpoQRWJWbO+r2x|>kIgWLsCbU>rjo&NQ|; zLY1B^w=yqveO+WDiMhMm*YWT}yNK35Vc_1MGlqI|NGB8LuE4owD%$C=`U55xh23VX zs0xT67%x#N#K~NL82V4s!sFk_IB)D4#tJ^g5SF{xHOQff{UP z=cg3U*MZp=YGp%Jwf*vW2@uzBEFou$s*RMJD)fy{Vw3AnY_cP+amu-dRX#QQ^rt#L zBdqrR1f56m{SkF_uC%F|Qko4!zmPfNR=K}h#ag;KKqQ&JSkqg{PV>&n{Rt-Ab&j;9 z>EFnoi%i!Yw7%Vs$$Y|9N2N2|oVmU=`h{1T*s3e(s(T52<1uK5 zo|zlRg+qI!%o^&Rp;g@CXPdrv=A(1N5}M{sYQ(gY#`~@(F(TmL$Zc_u{XT_e+cffQ z(_|Ez<|`4)DOrn1Y;!QQrI_TNc8Q^F{mciSd5T7=XhxHzISafA2t{|F7QoawWA+o} zC9+Jskvx8j8w(Yhuc-qzPJdGJI2D+hcA8CsZ%xstEqXW2K}wk~8UHqpw(uGmz|LuYNo06NT+#^gBnI}|gv{D~OMMb;_R- zh>}MIjr474)`gzDex+g7cpUck7{cGGNMzpkp?G6p* zg)d0QReTDRUgbnrNMng$P`iK77m{(>hS1mv-zTF$y!Z&kneZz=x`tT+yL+bMK8xqzqswOO4`Sd`l_iUt^i|w4hil0aW%}<`CrLh zbeDNqnAfcA+hBPgkUn-J+Z{=_>3QRksxb)F5S}#y@hY#~Z69^#z{{lYErw!YYaG|E z82NDlH+-nSQ!{g^=<`aZ2yl8i*2s=wiy^xDAa=Xy$ZFBb92UJY&_;F`u;4h=;>oFY z+L@9=h2zBXL*Mh6#8?*EK#%+Gnhja1|Kn$EeKQF2#2TRwYM?z6NKn!8005%JZ;~DK z=Z_$ZU)6bS(OOH!khN*6;@@w?#VEXR9Q#tEh}YjMuPJ;UEdK`_DL<0gqx5_@4-Vsm zS_1;UkHc0{7NDLci`fEj>ZQ4>v2cn!xPgduL-c)RI81oe=H~D&{SHc+tPqdnzNUC8 z=PN;m&9v)cHFgHcdmBvd30&`9h{E2eQ#@WoY99h4&4MubSyruVaRWdd$O|u&_a|!| zgd;+K-j@S)R_w>hrx$u2jse@s)=!?x2Z^$nB2mP5-rNzK*WY@j3$y!K*27E+UAWk3 zurlZw*p4HD7xZ-BVR|XXpCy(SZ>jyYTvESDTOY+WMt$(Eu)M-S? z(aLcw48}0{=~*+-GZ~^ICADJ9-$2%Wh##^izER>zWUu!jBb=1c>P7X-PNI_wB2b;y zVXCvqu^R#QqBsntN^VwykD2gAGJ$yjpy053b@^<*J}COBAQ=oi`@QXc*!XC<9rE!^ za0l+azy={@GhUy+F4veL_Z5j~9JaGy!1~~Qc!yzoyj$^AS3DSi@AcbFfmg(2P^8=c zq}i_{nh{Dv^2V9nzAk)g*x+h(B+u8^M_zp!iessKY~E1Qb`f5}f1cBSnF~ArPGns+ zYPITNx(0>Jq%tLS;6}V$`Y@T)HFR*FXrMT2>zTcLfipm9-U|SoM9EuZk33&qRGggMNcl>0S%^d-V!w*D)A1X$r?{CGwC}>Y#s#nWm3VpzBxy zRie81FEK7eQSrZutdzpHPuR+_w8%km?pSTjNVsd&^>k>|>44PbRGexTBdkGxtbn_H zIDy6sRojyn@QuOGEMmLF-78EYCMHYSZCal@5+cY`^*OSJx|+BY+6nOCV&x@FuK-zP z3W#s3hN{G~v(}DvnD}$^nEa8R#|bA=uEReDmnD!7!y>I>D+~5}e0L@+9ESUs6)sje zHn`zeQkk7gYe&KnzQe3?1D~QndfC$0Nx*GYYFKrSg5e)UKTG8=HVCZ9JIr>vIJDm9 z!KA5MVd7z(U}N-bjzb4-6K7|ZX<+zter_lEuCI>n;0_=UWCD3_u`>Jq~}tXd6KN1?D&jEzsUyC!x0M4v2E?dkEP@~ zC44I-3`rF*#1PC^YfwK>C3z0$9!ruRHZ;s(cPk)2zYJHqzxMK2MGm=m>)7Gw*21)8 zeQy4?60(D)kO*4FLO!HO>kIts>9n;sTMR5A9o#mJF^%~i?0{ce`is^Kg4zU`;l zt}mn-uwCKl#d$dg_#ezoFZt?PSP*KTr`wim zf(iGhYXyAA8n{-)i7WmCQV)CEl_%t0yt4irZ3VlaY_VjP zyTO>Q1Pd*ENN#UX5z`WqXE6tMA0tO!Fe%kfLlC)g1iixPL@YJ2={2yE*Z90C%5@8D zqQbLmik9N;wexZ%e`sLhWq%^qL!Mj@yIwYY`h3p5cYdAi<~NtY(P$I=WLBtms)kE}8yipC_l<0dSTf+>W)^@?) z@9rTg2K-<$tv|uOlMR;|sc4Ib^Tu52+#S=~*f6sp68a#E?;EdJn&j3env{C(y>~Hl zQvh>C0g>lNcV-zfmRM0|W4gE#Z}eWDXa8oiX~DI(^~+ZFfY{dq_B-hno}dHW8g4@; zo^cBC>$Q^#!?}%iV2^#c?h7SKQI;3QV~xO?=0Q)$97jr1;nZ~BxvmImh7dTB$Ysn> zNFLjBxyD9jG%eqK+MgH-eizmouo|m|6uGuPmV!#5J0!sOU8wq+bj`$9cgrYlQ+~r@5UlOPw^&EE~{{wYTxXFF?&65tPq{cCrWP=%W8d7que+s{Q z(&RMzS9^{o;O+k-Aa(ACn<4L#Lb!}~{vLFC>&;Jp{&oNR`>Q&M?mT@XzKtcNOa6&} z_SW$8q?L4zD#fW8vpoY#+(MLr=!f?6otXFE#+~&^t9Py>N!3 z1oFJdGLpDd>;LlrvLz>pN7*~eQI-qvVAuO=^K$d~lr8q_g+^GJ}eR*WT`Rs*ZM3zRNOt z)890cT%h!VNKbZq_sv*)cY&28KTk|4m;REQ>vp-g!R5+J>MMQ~sk0x(eiV-uSGp^# z*NCFAKHci^Wpx53wOu9dI|F>(_X=O)oh0q+xXVmYezhbI@8h=T$lnC2gmNt}9XP^p zoVz1#6YkmT_Dr(U|L{N+Pe>8!TKF*3Kez&*S(x!(M61btiN+2(HO?1y(zEF;Y;G`S z`0q_nk7p`uN|!=n?kd%2K z!4iyDK{YQeiYMUZ*E6~*mujNoMT~Lkk$A2@o^&VfM3#NcnBmJK=%p;`vsY(a98+mr zBMi1g;QXP%Db01Ux~np)o1vgWL+_gOAxfX+!?n)CFR;+{M-1>XEj$o+xPC5xk!& z3?>mO%l6l8m-aZVmWjXCWk`OVy?OeuWgoKy1?ndAM&gw}Dd(fwiK2K4WgeC4NG%{2 zn~i`xHyp6~=U0yMAaZE#DG}dQ?R_Js5=%x=Pk;`l;u7x3ivG|<&qtSWa_diPHJNF< zMv|L>hWqt*IVwsm*6_acE_HWm1=)@QDnxa{2y!lBZx7XgA5pvN%myxRI(aBK^QD z5@^IVzXd-aj9fA}@DnK5v-37a#r%^dNt;=)!98PWmS8d`-xO%A5pvJ-6l$v-L69b% z;Z$BWBi2O6TR$87I=+A+&uk(u6dfksBBWa?_*f^dzNL?|R~B=4^d~1hk@(9$sQM(? zO;)iA(w-@iVguqk?)swnXrTJ;FTFttwX*(saWFeSp<6u()HaX^N^lu}^J;5G+EM$N zF6%~{O5AT%`U3j5kauYDAfW06<$JnvX?LAKGos+T4Ix?25-+RNGXz}AymS-}A)43! z<_hu{n`Jrb*Ekt`*33^yIy$OmyN7BOm43-yFCzc0KzzMsVmtCs&%L-~ubiB(FjJBHA{SI52p?T(*m{N;(chO%szT@9$k36vclzR5W1 zSr%kNq3TKcORV|!(ADh>p>dDFB*%r;FDbdTGlR>ZRUC66#`mBQdMKah@Ed#hSUn+u z>~z(Y`!Qp#pL^UZFX1s{3Jn@|9qcghZ(^0l#9AOi*tBXG`D}eEE+@b27T2=%$Z?P-!=hPtab#>ofybq(&2k4OOx&^Or?mw3QAJeZpRj>|O5>L5WQUnE84B zHXv3nFbx`jV&lEWOL@toQ6t8GeE+{Z=~t>~xYNbTlG*96@xl92U!@KXb=}RsigGAh zBY<-a_!TS}qZD49Q_}`MW<}6`fB{T+tjzcGAi0at@0VZ2NoV1ogjX zNERx4n8D|ptjL9Oe%|K0F`w@}N1}qQf@>ivDJn;WZ(|K>*yTmk9uG`nj`@+i-CkxX!^gGvC}51L`cq7x-+NX-gHx9IxlTMwLKPk7nGR$yx_f2w@=NLl z#?!H@S?;WH1#>ZG|AnSU>s}_ap7qg7ApeP?6=}>4xfg&hIbz;TVZKJgzqQM3tV*o# zb)OP_=}}ZVWU5%^(ZN=RTztQTBS_gMk(G_gZx}S@$GL>@cL1VqnM_+c^RK~bp7C&M ziArg;(((ZGqH~1PA~-ss+9M1bYev-%91#D3yqDuvK_&`@+u1JCgnslmbs1q_KKSMN zM4>C-+au^ zFLWHIULBbzEg2%m@7{jCalk8V$ckT5W%BqhlUl;Am%FscWztqP`=4)HaX@7VNL%m+ zlRt1aQj&Gab5P#?h`;T316e_SoduF)ZV+BL;`zDBcb?vC9D6X`y@dPlry{G_K+PWz z$s}JQ8arV7Y}I>8Jw=1j<6y9w%WEm4>#;^sZBHBl9U1OUdezkiQV4fmk@ezU8eiU@ zawf|U2?bCqbMH>Ly{?1Y`<6^*PMafwD0F6ZOm=TIOq=ir8Pfg^V8kOJNYVi7#Q5;W zqS3tQ5(^m#J+Y(Y0!7o|&B;Y1l{5Ge+DVkqG9UP8|3Rv*(C)VX5h7jjmxpeH_ zcRSP*BbSMSHtDfKe;M<)nmN*AGeZ@mZ1<|{G_@N&WgfL(H~vh4s<5AMZ4qJAk7;Yt zFuDH0{8H_dp2A9&;XMLWMt6e-92{0J^sbOq<(hm0hK$*(?vXHRl@H&K`B20G^|y%qJh*soQ(B~d~-_~2?x}u@dB7K8kFqurX*ocMb!6D zF|EzJ;K&5{s$glPplR%HeqoVQ{FIYl94jq7iW&)K`MCPg`2Eq;&9qjS8l`;k5yIJ4 z`d9xb;pAG^i!rlnaD9-rQ&GE62sl3ZPxU)I>z(n$(HSN| z#6jSZwc$5-2#O~67OgjKq!1837zV9GE%#oEvWJ$D1GloF(HZ`m28Ay)4h)Hmk5`k3 z9NueP9wBvoAQzlnVwX&tVckOhB5ij?z&^x0xt2E*As}J#!uE!Gqch#$M-oK0Js~_b zLgJ7eYLpY3LQcUrM-6!*-O9#jq|?5%_DrQ=#*}7=>G*e{Fy!YmkAa6knpsy8TODGr~28Xq|9({JnImZu{d<~6r|eQJAt*V z_+6-3rT%Pnc7b0FaYwt;GC$oIbK2XvF2k&Fa|*3EyEh9SDJwI()?Aa*KIS3Q=q8j( zXM3=J+secqlmqhPPrdkSJ2iip^CIhy^Q50hJTZq$c-OVKy6oNxO8~lDEd}lKCCTRqi-XlnTC zt4|+v>%5$cOttPDJ=t%cn$xh`&UZ54{I$1-?9jIdz{dKoc8EC8hSW=O70G2rvATuE z^`deKZHG2x3~n+WjybrKW78w=3KzSSo^>w!l_@IwR>I^-#BJRPq^YZR+f-pa_B~=L zKc6a#e(QZHFb}&XHUw3zSvr|XO%!UAhUSS|o+*KdGubTZe;rH=%SE?&mzCSCxeg#Q zr0c(-P2K+kOjKscC^`dx{phz2k-rU?f`WL*4zoQHDY$X}RZled1kVglqEhpne|t7* zCHo%i^~Jcae48||%kmo1ZKTxAbM{mJ*zgG;8O&goM3pO9CqaAi|w=M)^8{_cc zlHt@w#uKTGn1ba29jy1b9~RleX$b&-m$j*`Lh&F*Z`Z+(h_yef(CIJkGznS)xRD3M zwDX&L3MvP(;{qG_Tsrp_l~(f!6=wtwS*I{(O>u3n#*N-hlkv98Rh@QEcbFiH<|gd` zv^?LWFE4y*g2EDoFaqFtjMy0MXai|st$YEVk_wI<;wkBF?)XhLfdo;|YE+4qc8H4c zhM4up(qHJ%% zZ-A9+PKD4qM{*QpM`LjdzA!K^aw~$4{gM9GZyag=&!dMw1hmKLl$A5>(K3OrM9)ph zYg>S?{$_b6eZq{;AuAbO=4ajI$BJ*@b9JW`;&d||nG%@-E@>JZEj{S`1>CjbKMGdeldV7 zO3np9Bwabj0BnR80PuM7Q_{EN`!18FFqezB4F62+m^hxf>;Sw1(;&~2Hh;I7oZ@Gq zyY-`$FUJkyt@`duS3x$Rk93gr!Jd}`|A;C;F+m(=?14|H7Ky2-i)M5Ji0Rw|3XAgG zjT0q+*wKz{<(f{wN>eWMvO^W}z?JO_Kq>K9yz89?tvK0B*gu<>mTlA<2=T zdQ>ckJ`=!=9e-4biOYZ2-}4UszDZudnw+`Vxo|U+gB{boG^;_>N^hLdIH#Q`$*RV! zJ1oPVZdB<|^P(PUvV`=V88WJT9x z))`K6g{(i8%>pgVFAxY^C>FpOTffR>It0OT2!6GM(|#7JEvBeEzxkG>jUUbOp5=eU z?1ldhE+!Yc8sco=WuT?7S^lz+&BqK;dG4QLqc&My$=~nGptWGy!rk*XYcut>K;agC zF>1$(^ae&gWfS9_`@pa8(YTSc5DYlihSLKJQwLv8VS%5Ff1FkKo4uyfcIT?Vf zC+99sN~)Hztd2ecyMHvEJ&n+^Sf!kce$VSx0T8H#Ba9kLQ$TwqcC$TibN05(jf;~9 zvQ0b75}<>2b}m8HnoUZOvHQ$<{&2Tf@;(sGz-)=M!yU=vz4eFiT0khDV z?f^D~XVdx9!;xF3RTA(qdLi>-^!ZH7xyUVc=tvRT!-7VdZcoQBu_h>?PJ0KQfZ&Cd zO&eZ?O5_{A!u)dMQ_+g!B!p?uFBYaS`scwm_x>;^v9l{`F4?*H&2g{lO=FI07N3ik z|FzT2yORq~qBwlcky7u%#~ zSvu4DIRCHskYuk9GkH&C9{1jFIAvL`@Y%h5nwE``i3^uKEk5&0QxyLwRo3T@0Fem` zNS{uy-O0f9TH#XhgBGp2+9?NGVB`%j_F=XXzPqE1u#LOFt7mh{lQWI${yxyB>?@50 zRR;9`jxpm{j^kTTWXSKA{9^iAsLrIgl zQTFaH3F(j1dU>bSFx8taZFel-S}-8B!tgh>!gcD5q6X#$lY_BmPgHh7<*&;85Ey!Q zujG@d89QDL%sn~ATXbaCJT2fOdtv0cTNr@l9~#8&6rwQ82P2cBCe^C-cxm#eQwnfu#L?{dP%M@Rc`(Nj38o zh9XTnhkmc>Yp~7o^*%q%z8SJFH*eKVxg-F-sAqqvli{Nw0U(-QSxc?x1@VqDWY-z& zea~q8_jdUwoe6iYz_FhCR+Xy>F9s_=I;^?>?mwwBLRSwYg}@3I)LVSJo=M$v%R>E@ zWxuafOU|Eq&8Fs<0%bJVMi+YMz_I3vZ_LYAt$^_@*Q{44FAosI-RfeS&QVRaK^$zBcUWaBFlq$|<_p7}5sw)kZo2~%|DJ2Laf@Yhc2tHC}UjbdLs z82@1J%UoyMY2~%R6DnH9PQ~QHBKB`e46PvxBY0hQHStDr^VZR><&j`3@kMG7*P8^~r;Xf8U4 zvhlm^#Pm-NuMfKT{AK_tUd{pnP}`j%oXvY|%D$dm6R+GeKlUdxSxm4=NlUk7c24yP z*(EtCj$akUTh#lfHYP{?mY)NaS7jczSJi1B0u>K>(?~0X z4rp)hjmP>d;p?8(HWn$A#l@C|86>I40zAFD&81rf~Bsiopi*avl?D$SV`oEuSn%Unr{sXYp ze|t=k8QtQZ3%ygWEl|9}zM2JCsy+8bV`uuwt3XnQ@v#OW_)LMZI{Ex3^xWf>Og#}s zE0;D0+1grZyU)1fd{QUaF_ydnucrck8CHQd7j@qHfsauv=0FS&6OF}}K3ic{-7=<* zdhvSQQK13g#9OAA=-Fbn+I)8^m+Ub!{Yft2>p1}WFa#yf*X_m9P81B&ROEWA&f6Nk z9Stk5NV}T!a~bMop33wmYz4Vz%wTtU`0V)x7-m z(G4LYE|FE#JRq=aTB}BQO)CS+Aw5R|ZSGinahhz+uua8?MOU2pZ@7eW9gm#P0Ya9t zL~2+6Hf;g*XMpO9&k89Rjor@&!W}@+p|(78i0902Yv(hAk6?f>q5WdVP4}Xcm~o#E zeR###eIX31782R12^^|;3_kev3Fl~&!dJYIWc(F~GFo60Fh8@05Nl)|s<(2)mm3wd z%XZdbhi`9+=3|dg!q^l;aaTxLVL#89%TQv(c9jRGL>e>)kg64C#aaQ@GtsjweX$Jh zCF6kf-Xs8^aoFCWzWS(6Da{GMo@1H{#_=WsJi}w{lBng9hN#1+oBN`kk_G@et^y)O z=FC>9eIgl8;zVZ3VCp2Hjq{#W)9w#XGY0~ZC!q&bYvTqA-Qw@n6MT#wgj^|1!#%de0%61*nWZ-V zXp}T3sO_eNjQg~{6bNUh&?SwCp8|4A`1a&0!7BAY|DC4FzY?@E28w#1_F!tIi6o443xZ#`69=^JjxQW|Y-2ih0A1$5Sg+-Y>gIg7@0 z;c;wA*L-l0_2zpiY1qjYW71Iy;VU2as(M`8$BXMRq9$9{+=roTlc7RnQw6yI>{AGu z-Bm=6S1prBe_*{&4nZAy`N8wBy@id*BpD|he9M`4LZ zxjOj-AlNvstXh)>kf~#@UFFd}fu1lIR;`3GOtbu{&R6@_+l5&0gNM>+OCSp)HeHn| z&DK0mXME!=Kq<9@va7K^{P*Sb?^k&pc`%cbUj9BL&~y*TFimm$l-N7;)XjJSf|r4q zG|pVtK-{;+fU`pY>@sbd5-|>maFPCM(}Uan<_5w}?%(0izx>pu`cpd(e%lB*{`2J7 zjhtS>hVP^^=XbBzG}Hs+22I7{i4RL|g;oUM00WVqD?!!0Zq&wR5(sFd0S~{I`rVqZ zgmu2?U8h7*QmVEB^Qx9YUrAk1=DfSZ;?)U`S1kw4OV5Cd)80wgStN(r)Z+##^rgK5 ztc!hwz<_CqUY!QePL(%m4oJC>W9E*Wft(5usi-~QK&pLn&h<$XxG>yUP+bU6UY+-B zfEWij?)Aaj1Ejwx^n@?CZVg-QY&uQAFus?v(@7XbmJ?>c*(660_KSi_1{n4FK&l~< z?%?vMSuC2yZI=pPi|_o+3nk4bkNgCDY_qD`2;Gg-Vr*XhOGfO;X$B_tgz#nBi&w6n zG@#E)pL4xbL;0unD=w{#_kXc-exLs=Zt+L#r;i*QPoE~g3%Ykibc^K;h4akK3vX`e zU7!oD7N9fjYIz-jr1fbMwKb)G}4rwPZ!Ua4TrS2&C6o8g*f)?%sO7^@d1rSeW3c_HM=glgrQ%?~?Ih z66{TFCMdWJii`EG>3zERsIU^bED!tsXoH`FsGa{x8gZr%t}Z;3K%I1pI5Q|yh40+n zAfji!Fe+4%gfh}-mSNNXm(@zg)STHo6<7|=_ET2+4hQo^yGR4-H>(n}i0{7bdmJXL z$bdUX>wIZ@8uzMIIZNELT5g%NFWM~}LbXkv;|qt0V6$Rj$>pbh*>Z0~{dTGC>KNRM z=Sx`#6hEY1y2r0b3D$}jxkx}D`=rdScevVO4a?fL1^jCW&%|=f+If>usb>d%&#oua z>ydGax<711dP*bXNbgblmFD)Q=6#$|D|czRiz;2+d{f`8i^k3bctZWz49+pTag%eA zrnmG^f{SVGTxb>ea2peHrJWp-z}1>iyo|!lUHL9<{{}&o(zE;mWV82%=0h*`!xdt7 z?uR=E<>a-?xU`z9L{ci_Pgkf+&6s}rI2@9nBY0i0R*qUI`9lJ)6h7z9eHdeOH`*lq zwR6rLqN{s*%yJ``N6i&xx>|~kqu*pi;iHx>-WsLznm>rEDCqtgTPrQgsJ2+}V`8(b zF6UUag+SeI?h?!%7ZbBusRI(Nf=sgcM)gZ*Vr{6C>9;tiy>rG1y&8BhIs~z6)n9E? zy2k^ReCy-lTSl7X9$b>UW*bLHYy;Z8MDYl-#M4_ zSnjQm+Qox9^MH4(%mfSAZe?AU7(Zf4zmY2!g#!GDovxQX{l30YR{~4-^ihi#`F;)D zR&-%UWAoeMl+?^SP^gCoouOm}T)!TBWTXKcnPJJ^qwej5``aHXJTCtJVB) zO0fT<;WF4|VthGrE`aJoAf&#&&xH|SO zHX=nF`knA2vT4P@i^DtZ#T|H;a&N;{Yz-gthpOl4TuJ0wh(;0iYhdWbCWB{VrR0*k zk>!53SANe_#L4DdSF(9`3od=JV1q&h@ob{K2z6~e_~-j4F7p|iuLA1d&dpJ7Tya|3 zk~x+lntjL3^%(ZmP%9|rK$K(+&1a^El6EM#zL}CC%j+(BvpMmC6nV8_B2}8_J|ENVoESUsn%eLw%UG-M|$+$vB;%Sn)#+8drnn&*MVh&6Yg0X4ZFW&a)EN zVb?#ISab6 zGG4Zcx*KYbR8G(>mFtwEHVee=GqdrCqUYH*H#bn(nIC1cRxKvJ?}jk0riIfci!i|CtK<>tmW;q^$%J@X!=q-3gWOWErM_1@(*qbX|@zxF_Iq_=4u8KUsy~P zCxjlXsjhF<%lP`ZKPF|>TOt{L9%VR2ij$`7>f<}98a{SBRGD`BaofS>rNZLqCOezaYVY=mnru7eCqL<*#R$K5l#yZe_wPGCfl*?5Xl;vY)3myN_gzpW=dWM)4LBGZjpC<6X{*A)hKc5I^{a*g}h?jNSa(M~VjGU@G)H7s6keJ$1A%-=yA>MZgsVporb}?VUT8ObQdZ=HSY<}VX?e2OQK{BpB`)nJmwcv z8dR4YPijNE&_#*f0g>nyrI^f*Hw8Ev+pku#_E6Xn<;?G@)__`u9xHJ7T9ZC|Shpzh zy8T{O&++yTn6fA`z@xxfj-1z`9XH=ZKow@{i)HF`%jS~j?=$3*fehYkY@8q`4Y*1Ca#qqv*_Bj?m9$e!5*h2x3NSZLI0y|@`Dvh zLN_T07CmeO70>gHW4lGkI$TA_nPxN#ufe*PT5K8U=wh~IhsEXeoUeWunP^Db3-gVn zLxBvRMwK2t8>!^5B6c}y&+VzHnEpH#?B;A3!q($)AkUK{Lm{vUrNl z+Ym3TaE<8=wj01CPyM~>>Bt}68WSdB*E=5z6;gYqpEjj>X$LhiYj9kEOyn51xZ30f zV5uC=9Vt(!*l-2;dtDHUm`e-4FsQ&7ebzL zJFdQ*We|PPVO_jx6ik~pk0lL1z_C?T(0USe%uf_(@~pP)#;EcO-*fe1DamKI6TY-3 z5N*lQNC`%$dixZ*kJ_wsfJh(t{qCRtKolg;+m`Ig`^GIlB{|_VcHrt?=e<)>^?U|E z$wo!Ltjk7jfA<^AQqDUMX%__Z&p4Y81@(R=D{NvN>oJ^m92Oz&Jq>2Kf$!k89goLOfNe!)za0;O=d9(k*5jf0f#5A=Sn9*WASU2sEZg1BZyYH+?jG^${z&vNfkUs|sqohiBcLEU}6$TH1T$8NjY8-4!g zssN~dK;-UCK7{*8u9dM`#s8}*TZplaY+^455jD0)9#L_LtE1LvmX7;vDjr<@8zKN3 zdWaSOvM}y!bYCp8MFwmX+?SImmM;EMVkcYx^?CC2AX{~y!R$ymJXqi z+6MCkCNTL%dUFR|`P!VUWZ)XBBhesZ8RM1APoHgimSeSt=>rV25WZSw4~=Xk9XcHC zKJgvs5q}(+FzQaP9}>xnKYFiZ{G5L?hw?p!3wmKKOV4v}p-o5-5vJv{5y)yZs7R-E0Q5&S1B2XX4te#ym3d zQ8?pAH9Sx?Bt64i_nuPlYRSYLu)_=13!Yr1%`x(8A65>H{S?ne!`;v=7(;%6x?o>q zxaQUlNn-s5{SyJ?g@8btbZHvDy7E-e<~HqRy>nxxdd4mlBaYL(&KHTqsRY{(W8a_? zP_i%YhNq$3c>2zp_WGXqNlZ)dI{sF8^8(NCt32y?KkTY! zc$f7}pz+x1SHCdJz0m>~=h=?~o>2XPsQ<@!`78+oCjJM3^1Cg-jzwQT=Cg#szP#Ai z;+lS^xESWNUq7ICz4iABS$V6I9gRbr6t<_(BprCk+TY1+3} z_@4X1KvKwBJjapiqox*gpce=u853(5zPEl}vkYgY;>&&as&EAW^u!N_J9CDIswGZK zm^ozFT_e`kfA$rNp0N>lEUcC}t^$ozuN5~hzhJ3TEdxQ`v)p9Ybgz=)sHa=NEgwn6 z92EX$0D)Z607Ed4CP&`0K?b3Yn1H`*FZFjZ5|4R3o(Os?Qf}6%U7lMdWMeGlv!q@= z=a`sZTv#-jt4zjU@}dMAS)pHwh8`?4^4NF2aTC3#bJz&907v>JR>+r+s5H?MvdXbm zlBC1Qi>7t!pTF_?u)~+%6HdOh>$Z1bnvnM^7G01h)Nl$M`x#k_qdOKXTIGkb{K|6I%9wNHJ#91zK9iW)lIs_ z(AoK3UmF-V!xY%{U;(TxlB^DqxM4%`KZEQGo7_2TOt#UP$*l^~v~S z%S3VR%-rQTwmlDr3K{6t&ahh12+L%BdGq*F<%IY~IX~iZ3AxTKa;}Nguj=;P9&S&O z0rX>$#~e^^jjTPnu=3dl+3ViD-o_*4!k9Il6DEeH&7iUzFAg9ywG6y>PGS6NWnSt; zC34VlZSijTE7Q-IumzxWPaiMz+XFmIXpP*UCa%9#4spMKW*p}J=(9|j4;2`w+4kL; z$KocG?ox}c?Nw6tO&r4OdY0>;bFY?xL)O%@Y-C`_+;QjL+;Oj7L-#$sD9nl!5L>#G zwp&{&bUgjl#3gev^ssBQm9g|hJP1Xta?h3gihE5_VKd}w*h-fj_CSDyaCmftuqQVhEIS?{2LR-G&y%r1D>XXR}CLCLU?TNi=Ir?lv-#m>NG3e zSzoNH97V?lTNUjoQ8R9I#0bKU>yL0$>ya`&*tQVL4hrymKL>r&x6U&2JaZ<_dK@ah z+TqwGh%X;Z98~}vJ3=jyKySEyiE1!mTKZP%07ZM=!okS{!ll$#=&?o4&5WLss`SIB z-nMFw?+F+VajZMkhv;^UR~=3Q!bAV?L3@e#3HBh6-uVzO+qC`#l~12d-#?)>5y2FT z>D6%7?%(?t2Yb_HILRrBEOCT-^}Q(#+V+7(qW}`dP4t_DeF*wP!lZH5TEAEy7s&Pw z6B>NoYYV9m|E6lr)Ac4$(#~k+VU8n@UmpS0aI7U-OR3`O3N1Q>t0yVVP)dJhkzEc? zz_=yO2$<{Jek@* zLb$IWl8t1|3iwZ=2o2n5gSIfL8B=Biv1;FMN7cndz5eOru~SrkT2n4!xy*d0F%URa zq=jK+wzO*Acxm>8sG_ga+J<}RLs-TV_6kz2Qaj-nA+~5`RMd58+4aFR`qrqZZo%A53s3H5L+wNw znF=qp9z88w*)#HS&RcM%_ptAVQPrvXsEOl)T^+;7Z=eODrvWXc>km6P;9~5UWBuHu zn#7?Y%h0P2ns1|-+vWV)x#L9=D;k_4&za|RN!dGS@R2_sBd$+%-Eu)o>5gHVPC#)~;M@Ud zz}rTJh_x@zm?eVCsK}wrC_fM6PMZ!}q`@u+TYr08WVgs~oAu79u{*quBcB|7bH>;n zD#?*!)b92+T9x0pg^kZe-4sOb`ElX0%?)0=gFTrWRK)BOmWH}pQjKfZ`mEMpKkb|V zdTJQEd5BNIN|(8{Qihaf^E+*+H)W4VD2MI^XcXz+*{!WK0N6532X|4EBmq&FPoy%x z@eSkXqsA5B$f+2=P$@wrnUUl~$9y#waiF%?oG7E%bpwgpuQ0saPZka+mE;~Ado99**F{1{l;I|e(CMYbtC05Rx#}n z_a@e1Jq+o@_wS;^0)8Tn_UXVYfO+)@c!_8ypT_uQzaUa+C|z8(FbXX4=E#5Q_xZ=+ zTR2f?!P}9kwa ze_u8c$B2wHt>nhCHO~h>%*k2&ei`#}a;TfIEcP{y|A))?yt7wN@?P ze#f)I$k?^8fslwNz3TjK!HJ`uBAv5;i`ek&g*MS7+op8Z*r+eAP-EMw+~!|=!wkQT+B2WH#&HeYP)hK| zh`|Dxlw%~gLO+*@d8(!zV&b>78$W&wk)tTo?A&`Ee-}P7OfBuO_o%kgD@$V9d#>(0 zkT@|xoIOuvzaX>U3$>n`pAQW9Ymb&2bB@{@(vur5a=SD=@!+eCy}DvYkviT1 z7=e8aCO6%MdhCi}*Ys**uxTOk7<-i5Qqy4PR$++tJ&~q8%{TfNiO#Xk-5zfpdt@f_ zP4l}+9JwFM$O+HS5dtb%4kdp5(jOywl=kD&bm5&J){s}mIYI!=5?bb=;Rf4dxVuOW2V0fAm>*yX_B6DDR38@gPY$1Rh$;EOzUmO z^z!MQG&(T15Rg!4ybC?lB08M%@@+;L3cjaX6Ugg*$s;l$S}97%(gq=W^rh{xyV!aP z-_J+c{0>jQ4VX3DY&~LcbMP6$bjuZdSoD^lDKPr_rZm`bw*C0&>r$ZOOrzn<{2|wY zSD&g06j&&;haaHH=H7=3^An@|kDUT9m;&NVg}@nU;k@?INHO8z9e8u@MVtr4mG(bV zKmwj;bon|lH|Z)hXNiYJn)0uXvzL;5XaC>y4*ydNIr}{LwSQ!RHx8Q{WC)w%gsUK% z1^VahJ_1*#8;v93D`um)=P(aGVjL##1U+s&!upCWN+|lgxTkITRn~BATrvDj;jI}2BnuMZC>k&Rm|Mr9K;PTb6kjl4oFeQVi65lVD)i`WK)TvsvF5Qe zf$pjmSho`3T`5*{<83$z=PifpcKXNbc=>}zySyz8FBIcW@+~boqC`t7@o?wZY>dIZ z4TQ#d`Q+Ri?UB=?yXoqXJGw9~?p!3CRYRA{2wew}i0-ct1{KMS;U*2G*Va8}m0?Fc zxawoAF%*Ow3%m?Z55vnV=y4|(E zD&5lBs`UkhE0THXur!p}1qn7WMW2Y2t&@yuT!UNL`Zykd+vKNPy~mTG)3L+u*qzF^ z9d|XoU7YhqEW6zcp%Gfx`_yRxp+E4|AGb@LVB4ZFJs}SR zxS%sS%bQBc#?u^r;)gAU*ea09^wPLW)x2Hq@^?vz<`mYY1}PEEpW zrgN-Ti$!xM1`M?H^bBHsE8Cbf%{5qMf(R#<`Kl_n^lV#O>C?m-6Vo{ZNlou*PP9Jp z_(-8mK5luqZf1Rd4Q8=G+eFa+gM9CBJCvaRbKIRa;mE_M!xbhjSp6Dyq#5)3E5w;N z%bK;Q+js{rMC_ltJ5U~}t3UDpHS4sQ`50#J7b?#M+eGsngb?Nk1OA0h#Yu@wn@*Fb zw33Q6nBUlXKTTfY=z?fX*>&V~-k|9*pH@CU?E+Jl^nCJ~7i&?JPQWhP^5b2-=oczd z02o|o9fAyE_B;+$Px-LR!AgOzQ)I;~;^1JgpW5Eu#MtorZSobuVUSd;LAXguI>zF5 z)0n(Cq*?s9tuetii|$~LpG75!-aC)zdH~slTb{@24xsG|Y}b)Mh&2MT7;I^YQ7$Ze zL(DGsN!vpQl?CGva$64c%Z@7)U|Q~#-lc?m>4Oi1V+-&GVw!V}%R4rBqNG_>!n$C$IFLY-lIZqY$ zE>HvoDd?*!CEKjfmFKivP}^D$Qp65SU~8?sfH=sF=;wFS;(z;Eaz1Cx>;)?k4FO@g zpS%=o)bhXjnJJ-oKXZ!dMCY{JT~W6=$w;<72dGWwban$~=D+9kX?NKWtL`40;$k_DlT< zQQ&ubob$0X!2P3YddVVcrRR;!O8_&oB4=Uzv^ka6o2H^I&@UsdFv;i&78)3L-wja1 zUUZC5fGwymwI-EF`Wn0#LhLnuKGd>?nepHHz5J7d|8juu9a|;w&3bA3O48`3$Pz2l zvB4`ND?9VEJBDawe(jxmG(!j~GvsnVJMbQ^XRnFb_u3HFs=C3li6YIBYhiDOMG-yR zKg2~7C2?2Dd5sX|E@Ze`Sq9p$hBjH5rN~Py1XStVdI7W-X87Hep(@g>83Xe7p~!T9 z8*RAiw5|R2rBAPY{Mr{8X&?U^pHePx5A^)N3bCzJB1tmw?mBpTDj$oaIoa=EaiA>RD^xL_dH7DBp>|fkwlPpp@0(FNLSD=p zLGfxQ^ozZg!5{vk9VumGk!w6ny;fgShO92?W0LLPGc{((n3jufy7EY-up+>2eEapM z@tfywP)IHEDh!B$jxn!3T;9{&)FV$@$L2*=bR~r`)*Lu_$!M{#T<#KewUQwXQnR(? zk2bn1;n}a$j;MH@TN%3d1EOA=8=Hn zUYysKf^pO@Cg)Q*NOv#CZBLQR*=L5_sb<=wHc*vX=F?JJ@5H`H*+RENHSonDMB&T| zV@KjcbYb>1ncNJa9*+UW5fD6Bi3@bH3HmP%rV+_h@0I3MW||7_qDoXPUuH(``2-SQ zAwSMRRk=9%`~Lh_m{|E&=_yE=S1*!m0_&k!j|E8(fE1y$3>Xp<8@vw>y-qme%a@=~GTre&ENm3#5Bhe=>fMp^0 z{&6YMqS>e%P#j(!5U3XxZZq9(yHr+gpI&(Wf`~5vAetyMrJ75c3TdXcNh)oC+w5V} zc9lhze7h?V^8H}yW6la0LEIpoJRMFAeM|U!3+i9_aS-nsQvTgX%GUyv>KsPlMvYZX zjUpusKtVPXy{!_X5vlc4g(j##rP0Fof;im9;M!qer{O*s00b@}8->+CBoC)UueKcp zo~M1iN2e*RMoF?xzrJ*#`6^2IU@fCvuy5oorRnNHs|a-k;KIWcI7_bJ!-BIze;CCz za5T{+^&(PP$3{?pMd)R zI1Oljtu^l`|5T;t*FrlqqwS|@uFs>pNqns{IE5}sND4QVwMUfV@f6jzrxa#2BGbWW zq||N4vkJGRP>(;DSx)cli@C%$u*PG=*2KqBfXtv!A`|dV`jZDLCoj@xJ)&%E={wp_ zYs8!v{nAXOPm;NlN%?BL8*9rmZgZS!7ztu1K^_*;Hh9!4P0nC~GS=XA?)2zO)p zN(7U)=IIyE9AWkmGJ2vg#rDQVLSocj*+Er;;rIw9|Fp+{VZHBC@UZ~t)u?i$2iJ<% z#8#1orpDw9TEaGY8=CE$+Pye1vBfeoe@ybGlMZ4%4`FHAD)q7q0|_U#-mY>z-8lPI7)}E4CX1f0YXou_*a0{cds zINul%PCG2q`Ay2J>*WiL%JtOMsun;4CpeUWmg%*=Q`X#uE}l# zUWv8B$$Qm%>k|j~aZ$OYe-Pn6t{lnZNbR5F7v9Npggx4-KL{2)#qC<_9$$#xQqFGS zU6_YrKB^vVqAXT9J?p`D;9C4LzwtKDJInuyx4B)iRhdkG(%QTooKDpFqMMcr<1T2D zW+qsFUpk#A@OYW^nbv$vo2!T$k$mSLyzNZ06<1ra7>DaFj;}+AFVN}aU!fxCl7d`4 z+!{}tn1jKN_SeJdbtmMXPi5e>l5h5dEft7dYPsMnH*7C?WsT^eGa=17&Ix!*x+ z$6&Y*l1a9oPUUg5Uua2{kPVYu92f&6sw=0;40m5^^kHcCKPYh3m;x7+e)Ha?J3$1yKb%jF!aIOBLg zF^cVGgxrjzw$ZeT@B~sze4@zRqiT+$nja2Rmr^+w;^Ma^C^LbO9NZ!Q_HCC6w<15{ z^WP_SKcy05iwlh&eqvB-6~`*TyS1T$-%?g6!9?&|>~IQMJb@qfX5n#k1FR^LRHQA5 z2@6sV2Si2D!+8JW-mav~?yx4Ka@6mLXDr8=fDrpS#E>ZMh6Cevo+kHLx1uRvZ>*}O z;LtB|0KyF+_saP~GE>JKfQVXyblrL3WOS zN+Z&28q7Q+XyHi|tU2_#B}$tUPGXbh+30{a&agAnEwR1LNtimn)eh$5-hlQZ&& z*$;ypyV?%lmUZn`kc{n0J|c)q*>)g4-yvXH%sL0(^9E3BX5z{HbNfw^B(((JIg=qg@eKct+B@^zNs zLi3%(6&K{*Pcrz@bNXRQ7a~g$>P8mkgwH}2%llM-0#A~3C%%C)bD7%9l9~!qWc`S- zl831^(g+(G%;HcRr18pc3JE+F$t?ijNdgH*S{=871Y^dh>o8gVQHe&Jtc;WVyw_rc z_#B-;=)xtE>*$K}&Jo1yHc_>zGu}ZhF^~YlbeqMZlH1W}3k{yVoY7q^$=q5K)Fij@ zM{{a53%}4>%?g6Wi4F-cLWJyT2k!9MDec#5GJ$v?L z23f{r8`+JqJl9zI+`s#Np4ad7eSPoe&l)rHp6h+R&+|Oa^Ei%c7+fa(?T{TJ+dQhW zU8S9qF0Ju+B&n%Ydutzb-k{lj*jC-P{UY?d&(c?bYeKWLJzhA9r{|pH2x6$O{s0j{ z>v#@MF!4uogZUfLq(z1Xm7nvw?zst0^TM}qr^cFHUo)5X>jM%zfdU8DAM3QU9G#$fpGArXz;_n%YAZ; zcJ1^C*<|PoT9hWF&Szp5PP}YHO~f$YP>R*Vvg!yAv;8WsJJp~` z&hXyGVk@bYH2lddao!wx$feS$nF=|4{Ny( zb6Yxwj`3~8T{}igcad&b^VY3v#>`j_1U^=0&W8-q^ZL4_3)Sk-yAz`ZSa!y%RP+LE zt&4Y^ZnhDyS*wMwz>vnF5k$-xjrC|R3Z|DAOA*=;U9hHku?b{gR+)`UfkSQmzf1#Szp+y9XwZP+fMeL7k1eM>E2PJ z`y@mWK^o7{MQcjBW@_p>IM)`O=@Ts7ZVl}sRxO^)G4>7=OOn+6MgG%}*E#VNM=kp$ z->tjFm$e_Cpj9gN=oC%A-@@2n1OwT{J7{6NLjIgoW_-HgnBGLZDK|!XsxaVKOM-HR zSXqLk5|9M|B4rn^tInCEgtY*wBMXz$@pupWLPbNs$uywSD4i_Sw~ycBrb;pY%^yj8 zx8t<>bF%N9`*q|VX)5wj-M=o0=a4f004t4xwwg@Ux%=8=bbluNdk5lYnIg(ycE$+JXTB7?m&eZ_Yfmf6)mNeJcTYk_l0v&y-@PTm$x*IMoy ztS@+P0O9;9h3jSg><58S^*5huOFL;o;eOZf7@DkpD@Ww@+<6#YQ(W0MM)7&0_tZ4; zLg74*M<7A8s#PM^hD?c{M*Fsw2#r2}#)T0`G>sDH;aUQqwXe-?Qso_yx#%qp%N=2n z3EeRn^;4ogsk(H{SS5f8i_M)`1!fI_u-mE|yv zcF6am;aRcTUpX=^_0h~0n>n!>7YV? zal$o|dn0`18d*CJ(mC-d0+6#)sJUbx8%-%jsVrj#DC#hey1fKkc^e(({7U?XzfoaA zbWyWy`7WjS?I1BT5HNh#J@(6WwJS1J5!^nH`2%%MtR7|;eos(4R-;Danf3w$> z8|X4++br)ebQ~L@#DCUg4ae?&?|~rczs~j)KE}vYKK5WzDqxUWp$cC)(OfG5A{9;tnt8{BuNjN@%)ozg_{P&QkM36%Tsye2@}bFIpo zttrnOd+Y=3tec6){f59cQxRJkWTNw?FfwvYjg8iwk@2=TVPt^Nl55s@RR3L zN!`VXGP=@#lT~eN^gL$betrGgREw?)21t0?TEwu@uBN5Lt)&y23s6>W=o}}F85W=% z_o^2=WzTn-3%c_b0pP*Y!-r;;-t)SVU(6%`DH2IMMHnwNxb==kIZwU2BufSTVTug9 zB>k;jR|RPqyU8Y5*i2FtK}Rst>9E=Ufk$X&=BjkwVvv#d6tOa4s$!*!aai)q zk0NpV^ubZ!WrxNIVwG{7QJJW&xr$l9o`XO)9V?`_;h{WD0ZCOs(H`gt_*%4z1!f*_ z-oXqjW(F>38?xi+1tJ7t^r?ip$lO9*g{Pcq%I^NBzI5i`*N24+tMZ9tw>X46JNYWt zwX3+`2v;Zz6{W$#N)3<)1)NV;nx_{J^b=N_Xe-J-l{6#yNMkgUcmj#{wKP09+kV22*G<)A&M4)x z*IXbSyuxJX0erY~7)PeKX*-O1a54#??q!*_qZicqGv}F=+Hpvv!PNE~ejsLpUTy1( zii>{w-ukqwh4GTsCy8JKFX0&o}GmRuEej z91tQ_AMFHu(U>xL`HY(=07cSX_9ls-x=ev$zQ7GeKV6_PS!vIAA-ex%EA{b>XyBT+ zrFJeuiIRNg(|AcuY0fI&d*sdMssmxti?c7_l6cREtaVGIG~9VWM8kR6hGQc^^*?`| z&k(Nc7sG{;u>?(z+R}P0t&yK*eD54$fS%C@Xbyjg?mV6}=ao!IG4W1Gg9|6mz4P!+ zK~=OH1WB;Lz7F%)x1BX0Pa2Wbh)JnjvxFvPMQJGU>v0jhrGDv)uC?YLn<~I!D=yrn z&9zh1&UT1Dc_aPtyqy9?dPZsJ#$xHtCW>CQw@@pcB(`RrmbwjbYulP(LVL1s-&gF# zFQ{Lc9;@i<7vyCMiri~m{WF`Pu033pQ}MYq=<*SMO-5qqb=h05otFTING-s_d+Jqm zYB@t5e3eHH85q$mif!m=B0&;g778PGvA$*`|6{SMeK74h>Kk96KC@t0S; zD5d9A-v&)Sw_{zCX@mz9)V+z19h7ThAVd02ty;G$ifjb;cKO)gL2$iNb^c6_?;r9? z3sbs_wdAi@9hzg)4P|~VCpS54)p*zC+#A#ZlcEu!d*{kX+L2dm9G4LpYqP|^M#^Yz8xJK5GiApkw=$4uwuzC$tgf|vLuFoMxER=1m(1n`KRZW z)5}C8kdWiIor)cgC{aK5Hl&&0(+n$gFzT3QV_G044(pb8!8fB8Pb0h6z1Z`!^*c1u zjeG5KUXT#TzcwaUDcR-T(@Yv}RM%8cQVVPP27I-^7FPpChmVGj+?AZ`?nmR-!89ty zgJSWqy5}?gjN4&6h)nH|fJ*V|b(&#+VGaaZ2S&d#b}Cm8zkV)EDt#3n4}OOB)B@ax z`$ww)cb_<73+q%DIU;#mide2BMzg$r)m<-_FHJ<<94|}mut+YrK&n0P%J~&-X`fmx zD&Fh4?7S@!GER&5x9h!kJniw?>vlVVV26iqKE|G`sPtX2n0S(DDR&D?^a_8K*px2Q zcVz8K@zJPnri@1day_r1O{=cyX{?|>;{U+*P$+PEO28N2UN}9+P$_ z_Q&q?!l#k4f}kBa+a$e3yy61VZ7y1t9-;K!UB)pN} zPSkv~ZFy{Q99C99cuO3XMFNV#sS6KU2|I*JHUbsbwPup^fTWEeh<>PS z(HgHv+guoI9yFs2pA?<0E0-Hcq};pbq=J%L*Wo?;%ZD{-kZ^0$2?M zNC@%+(d&^UsTMImU?mh$lOLtaAkZa?1{rD}6cR|vwF=jtdkPcL4_nTXb{+Ngahp1? z{*iL%QVc%zTmva+Z5zw8%kgx!n&_O!$R`18gotsu38j_DLJv>klnNqhr+O>rr zS~Q0ij(Fl1L8Tek)hR)CY~X=SrYd zZ_VoKg&q~!_Wjw6va?;>qHGZDfFQ{Mwd%x?4;I7W(+wNl3_0IluP~`W{2rSrkt4Iy zr%kgY!rk$27f3o%q>-3Oe$K04MpuL49nee!{!WF>Y_is4tjnQrE}bXsS2>GaAL-U; zx+b->fAXtH7cY`C3y>O(p%ZsWy2;v%hX**< z_w^-;1-_AzM$f-G?Zalyyoq!{-6cRlxU*Gt8SPmPBI~C2j)l(PU z(!v%!kGZ*05%>B!g~~K$+Y}YgCS8FZ34g(TqjG&lAK5;(3}_4H1)+wYEAt;zeiPmC zaYDbmlP54La$n55wP|XpdL>y12kDDj9M6GZT8sUNms)PIUT7>3IcIG4@OmeF>&q!A6&H0Y1ElOZN@EM|2HM7fJf-&70;iF(Wgtis{;m5pb9=T9RFALovIIm?O z+y1OLj3^ER^0B*hD=+-?+%O^e*tV{KG4+E2>M8*FbzJb#`3i?dzLz-)nm{wnP2K9WRLhGpM>)rmZ934jr0^bw0#JMNl#qVJS_ps1n=Cz30Yha^fg05HyU6T{g`Y zVWR&sCBn6aV#5YCdu$6xZJwd4~6K6NJzAWRhA6^kmwXlfj_k{zcLh z>+my_mUopU2M8X0S5O5>KI8dC%9zKatzHg>I8`!X^k9MsqGs9>*A9d0{)jak8+oNw zlx^xa2WGu89-1GI){6kw3R*e#qg8p6#cW_TTR?RN%C^b`Sz^P zoqJ^n!!cHukZ&J(FI8dssZWBkOwD2RiFeywxR9ccmi0oiGel^4S1H|t&@y771iuEkqB5c0}^)FiB6 zr{!pu9C2_^qP-O83PgqDVDk*;0J+&`^=bSIV6L+ai?ah$0{jR*Azm#DY*|skmP-XVz1kH6n-z`oKMD;x`x>M7}qlU;+ z4aL8&qNPWWXZ+*NDLvq#JVLd0w_Z*M!{wy5zjUvHS*)MrD&Kj>@84b3>75v`2^KEE z*CJ{nba?IVwAg^IRL;7B{@uf1>DoiB2jnBtsc0pk^=}morCg%{$>{Xyk*l?I^JK8x z>r_0>-Iv>|k=Ky0C)b(#E+MkpUMR08{1hc#J`C>W)@ zZ&Gzcq-!Yf8{Pb{wi@+Tg3Zp2?&%RxN!>ri_T)!+2L5AJ&S~FVS6mMd0%I1yfBJQ_ zafT2iWmdT6dygB(Z70UZfzS-V>3s7u6dN7n@0Xq2rX}al(exkVI(g}3xtL4Hv#SPh z2dkPlvaq9{$(zl~oLkx%DYhq^0c+){4gL_l5I0|)9MdJA3ba%*Sc$jNNG_)SGTO(7 z1=KA)0INrC_Tq3&oN{`Q27Rz7vGjtyvqIJv4S@X5#q2kgkmTo;7 zUcxnL^0$?GljRV*2;pcK5M1j|Ac8JzZVtP-9it^8>B?hA#7}B20ipJvxjEF00FoxG zD0=sRIn4B59tfZq*6RK*0Ab#7o-v( z9?11x6{otO>3x{nSa3;6dvDogRKhqJvn^Fok|CGM!`X1Mqt763IoxK*dz5=}eNB z7~JeZOlU=eB>D0kuUt=3>8E~aA5Md8(0J+l;aFUaf&;f#Bh{CeuP=(vYLk=QzjBN= z=z(rIH$PeSyZn1%*~QA*;nZA{boAP%G-oYNCYcO)2Oc_J2yM8?@UrI``LWC=E(UZZ zjUTCvDyy$MHHv@YVxc^f%p?4udRs*p>WfawED5`M*U-1`(phpM7*K-Xlpc6AJkZ;t zgUF52JByxqO`iYp>~~F>yHA@S?f%EA_^V9k&xRcZs+ZB+Z-L*1PRsg3lnV`0+uiYi zwb#N(PtAdu{u81#p>T&mi*boAR?nud8lRE~=V99CyJqH@vrz1hpiFOQccF0~63%3%8xD@rck zU^yc2Ha&`pn>C;#kZ!oHep^d+g#6{Z_KZJyUif@w{J_TTs!qRq`R{(kH&MrD7C>H% zz)!IY4i7*EMutwYGqH|5e=ZGYw?8Vqdr=4V`qi_il1yWY9WN#X2t&Wfks`#u(0m>q zr58JL*6@`=)c)DunFl(l{eS=Udiq}@udrhyK2vO{%@|L`J(Lz8a{zrfz8o+=_>NU* zk-X{r6;z@aNrdc?*y-Qpg;YzYz>fkEKjZIP#^pojUQkFq|Ix^I%=fJEZ&DLea9npS zAhux9x70zE%^T?Tz?+bK`|Yd`NB@n|eDy~5@GNL&qBi7*0E-z9x+$1nKV02_Mw(AP z-zMD#ind0T5q0{vfAlA3Zaz*X878d)d#W4%uFdxETKl!V&vUQ?Txg9x;debEo?{i{ zNunYm8`f*AGi?!}`rFAx`iyb@ePmG?JrX(?RChnMdW;u6V`&HA{Fhgz^<->(9SFUq zr4L(1qE^Mw&$5~qX?NJO^who{ucdqUELuzZO%7=u1 zp2k(8YId8Iup6ru*u7p4-|@c-zXB#Gz@_ll3JeEC*J$l`AZEDfUb0NcQekj(R0l;ydn8M~F&DP_X1t(c2LDD|h5H3%$C` zNfwHIn87GW{BC?HUis&z9#^vYrS5^2>L6j>Y z4~40DCV38i0_*;&T!%-mM(T6Iv3K9ZV{E57S$AhQ0elBjpbtm8=vh}OJPkvsKOgE7 zZE88CrXWns3%s)EX}wQQmLm&KG5U@?S{;2l^O369(3vUo)Qwdcn#8t6nirdAb=k{U zbBygA&Dp(4;-?|BW)9EL3A{2|Xq=ZPReiKJY?Aw#|o48nI; zlKQ{r*7v`T;JYE1z4otN;K>a?-qKtj7CSAia8l>xw82ItFtNP9D>HS>i*1tgqaIfC8k`sMk5EETSoZU+YkSG18F;+j9DKyIm#jK^@Q zIAHj;G2VJ&FmL0}zhG%1WMkLW88j=D_L~()z(9ExSdnD55b8R(>_@1 zx98}4`bU6#w~9S&D_h|Koc^}s#4FuV)vbDvCL~pe?!u9vnH=M=K8ahC+NkeA%f}H@TF&7 z-@2mbQzhq9XY#&`T2bxo|MkxYO~ID{6~kyuF``9-r#ZZ(sE68`=DkK=(o4B)ee#py z3#b;633@t1pn*0B5e3c2(}5NxZv!;le~vY7@BLgyt^?-mQSz@sbM!+$G&hKfekYTF zdy7<)o`u2aM@kQPAdt_AP-Zq@vmTf1{W=4=zoSr+Lhj!yrATsE#5#hfH`#Q5HU186 z|Lc$N8wL6KZ#%bmD0R1fW$aztUpC#jFkpQPB|RBC`du&y?H5fZj~&YzZWf-#P8jfk zJNsg-J*~|c?Aw?{Fq$)6v;b&s4y?BypWco?6tKWwgl>Dj)>&G3mA=_vQHkq~?wjb| zKW~F~|Nd@Ju5aXLAY%+SI!sD2-l0q{7PF$E&9Os4){h7k@LkpN*f2$N)Uo>OQlR2D{LXytT>>0%=c%pl`7cDM3Btmkl3hlw{_(%n-Td|=Xz z`DvLsqbF6>-6K6ss=eY2P@5|7VM+d|MrPnW3VOBQdHAedfDI-@Ild+)cd*`z&%_;jxCdF1{ zWDhWo@Sk~NVm0GzEiwmnPDVrULc+Hv0cE)KfU4TGmx_b`R&$3A`xxH64y=5o;G0Fz z5NvnzH3YSwt_6;qr%jgTrmagPbgGp2nhd{sZOwJT*i6%dRJm395rOaIkX4$JPJ2KR zrRw&01C(fh`CTp50@!4IRf4siw<;{!s?x<1PmxHqTapaX*ey83z{Ox|P!yxML_eWx zKu)W)6dmQpMO*E+`Iw(Q%FAp1HgYWsDmf1aeMFa}V6ChSD5*KWK0~5#R&4f|p#`xd z5hQ62y#$8Hk3txB-LC)9ohy683~v9f0DRiM7d@i zrFc9=^ov1WP}cqUWMA%MRaM1$d2ogq6_?^;c*!2vl?^l9V?BJM+6Bnp;a=sv2L1&U zIex!S@!0fTMhvza{P8|PK~4DdO{)m|QCiyRwr$B^{+C|le45x?J0OL|ng!164uxS# zaJxj3^JVcc#T2eEZ8{>04RvT2M+7h*IDS1m3Kg?2&{B<7i<-~|qLQGQY+VwqVNQ}j zzw{dXM;&Hm8*m_D>pe^(QkkCW4M%D5uNr59fdZwEi9G2i>S*~ESthVlTv^upQBpz8 zf!H|iSFXrfZEiaIFv@UarmSGt+q91|_$^@(cxULXQ!9~tjACBs&R>YILG85Fb5iH-Mya9%Z`xc^llua&|B=UT>c5ld<7V2X*XF(APr?~ zRFT#i^EQo_@?Wwey#D-Hm-^%ZC};ui=IqGgFE9=(X}V zXO`;ctDl3n8LFIc$odRzlE>YQ<2-ZuW{R{I=(n*2@_o$|fM)H5uwuu9Qs(SM-)zj4 zrSNRg-z%rBpaSFH1NHfybN{FTBZUi9rf=VRPS>5qeo|z~mbSuw*od>v=G_8~I57e{ zIft)`pfqqGIfplDYWT-4!S<0ByG&Irfn@Qr${b@j7QLhx-9Y$EHcmShyvbLE5uMrZ}x zgRL8A!gDT;W#3tlyOSRrk1 zBR1b0^e;@dN%PthuUsK3kKkFR@ZG^;ZSVms4?ong6fGKRK8HOijekz9+`dXZs&7G9 z8djNS_Xb?!sH+GE=NfM_EIEC3)HSqJkNEG)zqQK!(%5paFa@D#miB38Tc_3Fc@3GT zVBV`4Rc+0lFKjemJli-Uqwir=L9fJDiQ-RAr`8Ol79wYCh5^;OmiI}0cMHY)3RS7W zkuiePW+#)TZK{NCyNh)t-4!lxOTfHj*^9>Y2c1>xUt_<|+k1|}?lyIxI8hdQ(r2?)E_&1KCvDi(>9$e#ABNH5G8iT3=*#?8M0 zP4dkCx4fk_XMpmk18K(X)}z&32AfPk%0W^3@GIb5Z6SAvVMLPuF^SXxCjoK#$Lbcn zZ^R97Fe=aYggI7*<_NaG0$cbk&t!Z5+yfIZ+~p6q`~W3mHMuo&afMVjl>%ii9?aWJ zS8|Hr-$~&84<)5u3!~3oS)mqlfEEKzCc!2zEcYgXxLxL;4d$VI(p_F1(R_pbzuR?C zep`gHwHYB>grN+12di5KuHhIBAl6Hu!|dQN<-VPf;nL$8&KRpoT(d`v%6wn9$Af22 z)FQj%)Wl*^zut&^_~;o&qfVRc`-R53K-|g#vq6L7TX1+=z_{XD6YnZSS}>m>M6@4K z`mTh+@w~?2vq!G{JbhBH;M0xWIf4UaGSxhg6 zYnkma4;a#FpKxcrGDV!Kr3_atA^R+^(k?M3E%Yp8vnE-hlVAXI$}5FGzj5T4IbsmS zXVGJCn4QGS_MtwSLW$9wQ%eJ?g4D@&2uv=L-nqH=k=O4Vb4CG)r_wQFzmv4SoHquL zAH0jaRD9YkPTamE&2XicnP({O%`G0ZxbW*N>Kb9I4o5b#)bzPU>j=afdGT1qY$3Kj zBV1WD9ot&0tY(G1Pw$=;e1L!wcikeBf?vv=7XGcxy6#6V-|d|B%4N~eXjaRID$zd4 zlPoUfRoa&dFR(T8$KUS5LciyUr)HJcMnw**h?#k}!3Q533GHSqljfu=^ABugePO$PaWArJUW+nz zJAI|*>BUb*4=49)bh|jsH`tr+DAitNy9CfTELJ3%?)tQXu6F(YJ=_1wH{N@HxnHt> zz8RWDS ze09=DB%=8wBO;}kT-Z__?YRfyB6)XL<<#Z9MR5xMUGC9iE7m{z0Zqgd%F<&_pxa9` zhq)NdLi$`v%yecjrB^NV26EIz9kAg}`fg@v60WI3JiHms7EF94dOn2MRngN?jN-@* zh4IZHA8feo?@Eul$jJGXKVmV`7t9f7Sh66BeG$Zfx7c{Gk?F!&sZqS=wDOt?RAm)8 zs;j1Rm(X2cJO4mq1cQ&t8k!Tak(fonPUH^0qi9m4594aHohzP@x{=`^pW0>-HN)tT zXx4x!Wy3Gm*LTJ||3T;)9{zPYykJh#*|VN-KC&Ql!zSFi_~)p)T|Z9U4ucy6hnwpw z60d5wd>n*0Wb5VzPt8z$$R%(=*A>)V;U4Xg$xIj+TNJ^)J)sieyc^|f=kDy?ktq(x z9P95}td+i$D0V-A046h(!C9S%avHlM|Kg3&0 zC-Y`45>_(GE9{y)aE)`@xUcsbgwQ{Sr=*A_e6Hf9d}alXxA>KmM2V$YDh@mAUo6T? z`ky&A;eZ2W|JH0+41T_HE214IWW%eX2miWRpmT<>@dL-?v0aswY{pRUVThX5^?Fj- zlLHwks9+Q}j#?dpC9Na*TD9huQmtb>bt<;==si&dwyV{hZ)Y%sKa^|DCm|j&@bzIi zxMT~J+li1K6x)#WP%`erd;i0^QeRl5cDx3goVF!Z+Lq?F`>yFjp%~(u$~Duy#9u_V z|J=p;GA5lc7@FVUg};{?*6o;Gl7MBpc9~!UuZO|EB@MBeAm}1{H?Gr8J;R|eWd%3! z$$SyS%?5L}WsW9F2378+0%t1QZig>^CuM*3_RJhVjt*Vm?=pHTjO8_`0EhJjJ?R<< zm~>E*yQU=)$e0wK%sj-bSm-)_SA5c>7slBoG+KABVoomn7RQG=F-%a8P-n#Lw`T~k zE*G4)qO;C;`D=LQb8?5l0jq5Pi81w*vj>nuLocUw==Xb;|B8`|x|y`?{h ze}d3qG(H+S5*t4KAxu<|-zPum_RYtB_DcQJ`Z`+iQc^<>jXXoTGptD156R8J!v+=0 zcbbF^5xsME%Vr7|&)K@x0yx4{@&g4Kl1JK79Y`6ZIRhgleACv@;iFe!=5P(pl-ZxG zMtC6-J2n0wPim3l5_fnzb#R2eU!Y}ZB2AUi8zyAncj~GK)hn(1Hw#@**qPOJ1sa^; z`<9GE!2{O~k=>#*OWt@L!qOnIx>0eL9tMo$8!1A|ZDB zE(meFHMNqed^2h9v8Q|aI;7sRm`$Ng8TYRL)!y}YDH~s@seLLL!efu?i1I!ij-`!! zs4x1pG`XEev$(bkEscWSIB$&5Pbd}V%6JFPbyBJu?^ilQ6Ky(h4{QXL+ zdvAXnqY1uq4v6<<~ zGUUnF$JoYCbMLc$rjOB>P%=rJ4pk&M6~Xj(?aK zZY!BAitpyf@%V#f^KPa06H*GMCt^a8JWFPVoz)L_@=W1&HNJ;UtQYMib6Rn0%LdNv zl`w8kDUZ31*31-J%U>hC9g!kRdP84=46|8W{fyjUvQb9Ww+!9+Pkdh;e?1Xvl&kmq zy<0+C04;1cedtX99J>T)KV348B1yHXchK6(ks`ND`*0*4h+ znzfAT_4_Ce<_d_Q*wa(@e)H%6+K6_sCXx75A$SVYs)DOEp+5ww(_N}p;62!JO1A2hG4}?t%xY>nX~#v zu;AR*jWvI0XvtN!z6~UNsXYxd4xvh1GSE}ScD;>id-ZlH+AectDgw4WNI;*_Q@}|J zj=vT&B&+u>K%qF->nt`**^%u?f z?_Ut5Ts_&qW+Ocxsxgv9^<o*S{?kwU83x z&}U^HmaXAi0(HTgU8hsN1>Ji#{Q0aXMIp&{$i7Q5h{InWDS&Jo=i0Sy&?-G^NZK;Q*QEQ?4~HTzK6Yg zxK&Q-AfbVR9%2YW{!6i6OQ<2vFOH~%C*nH|?NN!z8Lt?Vf?-0UO5}`<;HV2d2aB0$ zJy47}eGj{+{kr2*tK7>;`&;49IHHYGZ=E+kXwAYpn*R|Z z*K}*IBA?hVC7)h2N4Q+DdXTKF|Dsx@bX+KmV-a>DGifO>msA6huv6;^%cK%F-IBoZ_4$b0`AKu&nKdunpthK_-ke<@)pR{n5U^(YrWH@?O z1C}YpWiOnL@ft2loc5T^FVpo4OmUG5`6UCZR52kV$A`89XDez|=HmC_{#LN|R_PDU zdjBtX^eClF<90-jp@!IE%m=?u-!J9_#o5-B+syntS0O{$1u8wyh`=?5oJXhf&Q2Q4 zGjR-OswtQ>8X%*X!R?5iHkvt8fOL>3*%8be zY)pf7&(LDyyc0cUIVnmUgs=b=S2R+j>xUhiF(j(8^b0XC<7y{l@aC zAPXX-Armh8q;mHuEcQG=y$Y6y_ilT;BmkVEx(~zTp5bH4`s6MDyq=Oi_k#FHzvUs{ad$og zQk;cUXu4ommq?*mvNHbJs1(EPwrBYJ>-if=`Ll>UFvaSvdVRpFR&VxqV-wev9d$K; zEPA^2emrD5z1)Gy@Xzbq-Lmzk_V1dipp{V~kr0H{nyNc=JRo{zQjjN{js=P&)M>8` zHa%ZM+OsYoIbkRP{fWu0=~x*`+WP^5#E`hXk6!{v$pu2^6$emUQj5;a4V9awiMvdS zx91NY_ZQ|rHEjTh!^h{gd(H&fZVSa(Pbo*I*n?xKo2Q31Pg`J;wv;o+pG+=ar#y`V z25F9^TslME!4^ZWrVE|Cbb#G@$1j2$oay+?4T;_lstw;GrArs9mWH`fz_KvwsqFMP z4U^#EhsERe-ou(|<5_u2mWtwRH6+Z_)j-<7xkA$EkCJ=3p``#edI$sKA%pw(oJ-Th zN3iHZ<3AK%b%}J@!EO1+W*Ao5FE7hZS~+ZI{%iJmr1p{K}cC< z0vDc?)w&3c&g*X5uW@h< zgy`OuG>hT0pW7hXOJir&0rX64HtbBN?0c1mQ}dKqXA_jWmVwtHZ~W4fE^#<{cv;#?EqAx72#DRUxU3>5 zMd#9|F^?xcE`4B+DJtZMHwS0^b~p%_eQncI*s9fi2Q5r#I1N(7*65H4qhH_;ss{N- z(Dm`jRVn*NF071iE(Hr;rK;)ffz`?TCK$X+-T7nasY^xX%YOO!y{8mveupV$zVkbt z>ELl~se)`q7aDy?4)e z=@F6sUefSv>CBCdz+V^A(^rrWE{~lnjJIr4VU5Ell}B1V^oxCO;dkQ@r_W zr3Kj2c4Ip7L-ywCuChL=IRHb*LEhtx4f#1;%4RgJK}`<87<{vT*2k;s_{GmlpoKUE zxi6?R!6;WWnJI-`V&YpR^K(U{jp zKS|1Z+`@lTS0UbF(tpZo3gs^#cNuGfVD%g=Ulvfq1^nWl3$>;#Z>*C~JU`4YDP;Rv znyRb72yFxf^>?K@44S2bO2WW7#FzWo=-dOSXM0m@8$Uu^fs;vCGryw&Ly74C^`K4H5rH?cE)zQ; zyHQ?Bc|Qh!nZVXHlyg2z;=7Tta-3|uR-kL}R+||jUFk>c%p;hrNkic)AI2om_iS?; zqAoQr9`Fg%tvM$HyEUJ&lH6sLkM7lAWJuySj=Ewn!T?)+R(y;iy4dr)tZI%7xC$CZ zb{r}^W5a(A8$3dcV`hajE2eGv(Dy8wKi2EHJpf-^V}T$Xjz%&Z^()OikxtnB*}Y11Z12 z4a6hp*Xgp6xY4yPb@O;+P#D1G1xE3-&d@c=b(Sfq>x^$WsudyX#MPV`R*9hj_sa}z z{5g-_q`>bmUefTG6rZ4t9NXC0sGY(8X@m^l;?*tS((4<(Y~djnES1O|D?@GjP+y*u z`p3+v_mC3PRhtjr@el?wO_ZA@40jT8CaFt+UQQ3uQ7>=!>BbPl2cl9e4evRI?Z~eA%z@q zSDOE{fE&yIaa%bB%c28*KNKu~+;4jL;p$EI8WXgiKVddq{u~$P9Xnlj^W^!)vtw>-4t*sp0WdH<%2 zy~mAtIx|vNi%K+Qq+t_MYIGu90=XmPp#Gzo6>P{COW#1cK?^Mu)3z%$*qzW^8@yE_nLi+_4gmqd-p*fTSILmK62YiXsPD;&Sy*| z0rw0qD#Vow*eVsx8bhS~1=uC2o?`lRr4Bv3(6w>M5UDA@h0deEG?M$>G!e>kVZ7ItQFhfIm~9B{sIn!3ze2p<2QBAnCb1Y|K92y6nmXr@ZFC6 zv<_?tlw(E4e~yEYZx+E=yl7#NR8ro~k_8CRqCF^OVyx~6R>i|>{`)fx_m7R`8c${b zLR(NhFR}Jmxm0?snJUt~tJ>oCbjf>OAq=6u5u%6Rvc+M>I<31lokP{-3^J{Y+UR44H39GOB zO3wRDLDfc}q}G346SFx2H0^;^#*iIhaA+G*pwyVVx3Ol=?r$v6OHeH8PVC~Rqf%mi zvxqRNetV^TX9z7{9+o?fzk6ABps%mnG@{u8abFy1Nz=d+Po{;d)?g*{$FfTdKO2~C z)_6*F&(`o(P4`e24ooRaH|i!>{nWWCfs z*t2$!T`{(7!dLW?i;bUEO@d$g5}ViDOX8@IgivR|`eS)SR6BpL({_Z}H6hApXOoUU zf>qw|^Z<2Rku&;}(vnb|5lVHDkwe6N9aNe>Zh$Ym0V3J+8NNx~(yO}<>%V^8x%@ul&VAJyMHoWumxKya%1);>yanPhPk07>%qldb&vh_(nmjXQ;b>l4*U%c&? z3|~tMRb0Kfu_)sFbA3xfen>A|@N}?aA-;?>0@camtrTDm@$gFlm`+97!N$_vL?QF$ z-YM6{n+JU^u;exq4(m?-TmwkK3o|~Joem%Zf6oyhntkCE;T;<#=@NhtWQ#t%Xc!VSflDYIYzIs0-pBp zXV1w3sqwhPx`r;koVsGI;odnjQFFxc%n4zopAMgDh6sP&zAbq8phhlcVh_6;Z|*0} z8(>BJpNp5B%$Y;UQCcjyDL)7*%PWB!+eZZ9SlS4UnR8c@!3B2@BfA&Q`gY0WHSW+!|w9=iMqX- zik$t;LHqr>k@yf1-A3xYqGWH*X%VhXiw|}3INSBKx%Xj)qa&r`jEo0K<5XDI*WSok zdnd`Qfv#01-c{610064$^X&Qz2VQX@*OzVmdDoq$y-bvY(J!(G=57zZceDeQ8lNmW zG=8cjwe&M;=vn;yi_;n9c6Gn#^Is6%`i4E-esA`QkU%zz_zU3~-Q63&ZWnRT_;9t} zva(ko#ZppVVjH3{AL%4@|8;(rBuY$j3vZ*Qa-W4jsRohD8PmF!M#9!b6qyI9v7bJ znsa5SNp@KgOhg%oz3#M5pUOvKvvKqnmdrAEzw?LPcM1|j-1@8>#aFXed;C+y{{o$# z@+r-y+B3<+uiX?y``q*H#(Xp zP0ynR;*2($F0&C)bBHOA2s5UHL?c8%$kq0nz^n)}fO6Luab@S)%jW^i`-w*QRKSLk zo<3HY+T6#=v79W$`95f*tXj#vkwI8hN=0P%m+Rd_WylGHI6hZPHxYc z46w2XJ$vLcs>#=!&{>5B@#~!dr&LFl&-dUx6tlAIS3aK1v!p225s4s(8g-(V5hHr< zqZ>vUhO?yX-`RVg^T*leoWIU-`Da}-*Rrg&p8L7)@Aq>*k4bVU{!~9}P10b`llP3_ z&F%He&JT*08uZ~Nt@OUdHb5@_)M4Q48yy!XR}Ps|cFDMPsE52=Lf~~Fl5O>d>Dbkh zrqcQ;5Dtiq(!ZvKX`|mVx9qjZvnPRXoJ*25-34j_bZbsh)KiPW;LN1r*4E zJ076JjpzDDtJHg^!ZQ|;^`h*$=|11X&u?11cR{xAgz#Jy)5_Hczl6+(0lz0icD{|` z;gf{fwI%UI{?{vhKr*xYF?GlG9V?Sv%f0&sK*%I@O>f}oCRzaBv6!HX0c@OVz{x7^ zZQ(5$7xPB$63+n}AZ{lFY=`X@6ZEFDfpvQ=(#j>o)Y{BC$}0>j|FLT7ZWhA2H-JVy-jOQ%CG2l=W4D z1K7=dEvG2A-U6{2!k9++194PQRXf7<0k5CxhUeU{J18`mlEK^o5&#z058f)C5Xs?I zviN>QKM;}I+gtuh7>D}Nk506&E|8ha#yj!?++5eUcJ~^r9vQps)MUP2-cC&YM`bz4 z^tuLMjA4%nTJP3A4^fkWl$`BD$bopFi}(8TzN#d0Y;7H;N@-jUh4n27E^?FfiB$!6 zTBY*^InAc;=+8P2^(!N>kdL;tn;B{6*MM02E}j;jF%vRz&b!Xcqu(++@5zf6HEz`( zYb>e#RWh-uqXi9tSvBs%JjBH2{c2zv?2$g?{xdPLiRCFz={g1=zQ}pAjuN~BM%!Z# zM)%RMHz-+H%ZKH$vKk@kkwntbys4O9UAqm=r3!|@vb|~ zAz09c6#Xn*9xD4538<9ETPJ2ZRn*~^5W#1Gy|skamZL3zq7nA1fjOceH#HQF-`7zw zf(b(6`Cb~x&#Lz?XleU%ytF|H8?x`#8@?8~oAspP%g}HOkE#o$?qtE$>Yxj|`lcY7 z`B2&Jl7NVw(yyPXi=TIL#1Z;+~zjR^8<2pH10KjOqA_&}|tb)mVnS9ggc zT7UCWPhH?I=I2|D2{HoP-#7Pgdi90LyJ8eh>c$dAyq2 zYqYz=rIg%F55}d7M+6>G3mo9~|L{0u%&y>rYd=}EBrWpt-Ji&xuI7)*ocFXD!cfJ` z^0H~Wqt9v*OjC}~1|4IZQ)$9U38!KN-B z0}2zlomHE-_-)jzR&KI`8`c#6{e1?Z*PZDncatVCbGgL7J#rAcc5 zsQ@XE)VKsJ%5ZME%>i{G?O}a(yi(B?zWQel03-5o^F=XMj?3u-Y)SUj0OUxe*YHQN z4}td-z3G@zS12KQZj)XZfeF1)ET-_?+L!IiqE!*_djMscdNFg^iYH1i5$pTFhz-x0 zrPMCv-Lu3Ek!=9XS+zjA5o6&>kdcjZu7V~CsKL~cZwxW8hcWQRJBjkEt6)$#0 z@t-om$^`P#S`^a9^uww?p7G)Y$f!aDGbsH49W8IlZmDm>Jl~fh`mj^>G9jgLXZ(_2L{bGG7?p z5%nKSSj$pBai|U5j)^tV%YX=41!%R&iTwZ_*TrDp(wl``)pCnjfna3!iE?YVC+v2+ zsx3S;{M|{DCD~kaN&jvz(0t>i06ChLV@g%y0MF9>0aB?*u=eaCxFBAAk$Z zvIsR6M(N?Mf)JSLo6NH2mZEjaB%_^2)CG@lT9!%JIKSZCx+i`0S;V`6h53BeANX(- zX{Q|)>MA9^T)33V$dTu^XtRW2;8MY*SqR;I4^Q`ARpPy;mtjh;6VTXd6(a|GS1nqj zG;AI``lH_xmT>{pGl1NAJu&6FP%B!uPx)O~MRIg|Y8J0+NsZgNl$|pk zAgcx?9{u}VJ88P7~p6y>{vf|_^^9v`-b0hxjc=(?hMEWV6@o}CkJrmTGt;e=&;2_=}P~5JiYmA=E^o-nrnE}B3;O=dp(UkDsV{CGbzU7 z+f#}P9bJzG-7}zC!~=`JhnrIBHn{YqE}qs7b*_*HMn!gY&+>ufs5`~`_yZc>h-mNt zm`~wxq6}a1B@OnyaSxsMG+_q|0o7S!6gDRk#Qh3ITlb%8TtUR|fX06XlhPT|1C83G zT&{zP{#Kf$9{s+$M4m=>>}-ucH8#HlPQLvc?>d|KJ0tl&D=^!HJ`qFite;w{ez|{? zb7jiz(^p`-6N&K2u^ZcrL7AP=6zjFuYGsXQTk3p2eik=auU{k;lphy6J9X4J+a-0c z9o-txcrg_`Qf(w@$pYNuXTMoZAY*4OCS1q`It7XeG*kO$T1J7|I{CQ;EQQP2TaH-1 zHkTV==)0#rFHqd}_4@{4dd3z1qAGv8yg%Cq;FD+aw>i%60gABuQ9hM>cfor%W>hl8teoh6m!vOS>Q$Ars#n}=bL$VsP-GGhv1TX zoF)2Uq=evOTvl}=Oum~Ch6of?o=7EMH|z19;@MU3bZNIwA60)OJXil47{p|`B>d;S z+-jj}C0dQ{Yjw9=!{O<&0mR&3Fpyl%#Q%{l-v4{1fsJzcH;!g$Z{QzdCgm3bfBPDc zmq&aSpJFY`-gP4eag$XCG~6g5FcYphvXv7MZCid2$JTBd=v4(?UwAAzf<(|{v!UCE zCCc56K8>Vd{J!h1xwW-?DOd_LzN;?ycsslyVDjj$tHye65#!Y4c*e-Vp;|B$HU1+b z$PG|-f@bXiUB|M&m;OiT!B*YB@V2LT=TW!)S5tn!M4oLjSAW+gka^c;5JQ4g(j94W zocQZ}1EqSrmK|mN`vJ`+0$Z^Yo%ttFxfoyUdgGG1pD0dW?hA20;9ZTXLWWp6`A%sO zXg7{=X1wxf2%Y71f!yV<%Hak|ms%S3_gbu;s~++^d~F5@7}+XI=a%Qg+H*N1_t&lP z^Y@NHq5D2Tofoj+f!r?mkv`U->ko-Bf4lT|rZrN0 z5L$?B=Tx>>c%yi8aa*lo@cYbi`pym&60DB-yhMue5PFb4dP4qwFrWr{zkMd4Kir#0 zHLh*x9uq9r7i5U!nfNSkCJXbFg1Y6~$h%yahXk)Tg~-h%kK>`NDxxl_(5OB_=f2SR_@)= zC-x|GL-1?Wps)5F7mwA#C&V}loq$uWK3os3z#)C%!TSyDzV1bcQ5nP#K-oWk!PT)> zGo8j03y>wxj`cvm3pVhZ>2P8WoM&|&J0Un?sk7N8Z7#Ih^t8TuzWpz!f+w+VXZY0% zS+w2e{sEIO^H&Q&S=eFA@#x{3BX;>yS*1n2K+6lk%OAuz*HvNlugWEOVA5)Q*ku*N z9FC|zJ1XZ?j-9z660Q09EcL=rjh5p*Nax5G@d>JO*Ki-SP{76n>O7I*4*yMt zbCQXC_)h%gb*13j4ud;Wui{Vix82xtW~6%=Q#@y6W4p)$7K8q>(&5{Y;zauY2&XN-Ph%>y|`Eo_%DVFl7oKz*&QXC>T`|4 ztOM^1c2twGC06@2l*VMa{NS_eISC8%hDN#7*x)tVwI6H)tcX?z6!f~r9#J;?Bsn2y zaHIBolr?05b){=+SGaa-r>Up!&f`ToM&sAA)t5y4OXmP%$jkD0DEi1;9kUYl4lfO~ zr%Z1N+ZfSev$!9*j^C>l50aP{kOgcK^L5-+jI1d+ES=5|5ges)e9|3Jy*0&lBx4_{ z3Sz>SwynYEt225RdgEs&$b06y^b;A2TOHJ~dL|m)M;?uw*?Mg4Hbski!w-mNBm3$} zh|V**&9z3(90SFHom@_VK>R%MkDYMZNYjjpTJpEiBuk(nrIsX#J&>NLad>vT-}~~1 z=_)WMK6L&gO+5YJsfD~~zo{OA@pm=~F_eLU2BUmRy|xc1q>7Cn3K+{Y3J*Y_A7IJszmfK}Q+N8NNsuKz<3GFadgF07I z`&ZVdqPQubzys4Nq@Va{sbA{Ka29Lv%ZcHP;8@w!*Q(!Du9vLm^eoKdq+!}}`Wo&l z!|Q`Dw2n}br&|DmiwzWmBush@e9sIBVxQBsLB#|b5gf(vkh5_CB#DD%!vN=?Do{8t z=lk+f%l2my7V9ahy!XwyDF=L&&c_O{d*w+I&Ecrl^47msC-3>FFQZ9?nbf6RMt@si zI@{4)dO@ZITBU7-Lf>myY=X-w2Ny<^jdWA>FQQx8nQrk;@OnYeI^3&-TY|kGyJai= zub2Dbbk!MEU@2@1hyX+M(t3Mb0np%GvHw4=7(=ehz^1H`?W$05;y^G!?NWENY(u+5 z{WHyh%HVN3)_wRq(6!)LI41*+4n$k0<-lJZjF_Ik5Y7KXo}Mgs0i(^#KP1mlBqP_^ z3yG{NleP;+E$OHS!HFZ6)$)b1e=$4iIO0kmX>GGIB2e9k+}fgVeW}0~d>iP3aOfSM z@R7*eS$g&Xgrh)J*mCm&EdZu(dr3(m?wmH4D~(B6PHuN4Y!xw~pXXGvs!4YqRq@z$ zC2Xo4E@WtBuxxOKF-pYn+9eUMjkX)7#z7+<@Q6~7%C zPE)~xKcS9VZBS_J1nP%th*L;Ei;N|NBMy`2smipyS$F*K^8wJFu@_iNOQ?{ny&m9- zqG{V!gURe`XSjUeo1op*Q$jNfj)(&K()=Hjoq-Akp`(fbEk!+mPPRk>`iID93Y?I{ zgr_KGL%SpBkzL3C)8e)=YmQ0$;_;gXO~qxCJ=K{IQSPpD4gAAOptor>D3=WI;I1*) zfc15dytwOpMBaPdVpkIM zJ(Y+D^=U;uGi9S9z?e!z+)%V=wx4Bqe)P4n4xZ!Z{>##xwn?*SdX?RO_T>jtE~AIx zqKh*9m!;$+o#FuW$>5mh9FiHnN*5Q8~dDxZ8+(mm2&=V@d%xNq_N`4K=qo8rC_G|eJC zpgr_Brwk&Z04&0j3NH?BSuwdMeXpe%aD8Il7~7(($n}?j0dx&nO}2fEj>da0 zd+Jie`=Ia_k?wQp_9ly-B@UKhCl>LDapJSHaV-5~(AF<`zErqY{qj5;m*M))X;eUm z(UkSfQ}>=sekT-7Sl2nl0dy6ccFxLhn`aV3UTK-NV5915-m)EgS*!Q&C7q;W0VwL$ z1@&b6{e0GTJ5;#YLCDbbS6h;Pj4kQ{iF_FI$al@~OAQ0Yb|1C+lXqvP{$sks>k%Qh z(K?%~0spC(H%Y)&F-KR0s>FR>3zxO}blZwN0IJx(9%a*(IfCrpWtqzmj_gG`t5!MEvYJ3>+rUJrBNTVNb zQX1ZRj#zROE~nHN@x3&XCXsG9cF5`h=PLg&b3bB&nm>01t_@Y&7T8~jT<|^S&*+>%-hZHGqRywQOzdJm?B2K71Hn=`+Mu7E&L%3ZrQN5rycekI zz#kp_#vri#Qv*r7lP^Ozq1j7k_y#$QT`>9%VXS8t)#h99sr>1ctPGC(06caNrQW2> znPJ~7_!lU0X!P;ee6RvB2`ky9kuAX^PKf-5>gw~i7N(Q%Uk2!5*1Yq_R5`;m&k3WiVoQ-J zvVF$nZ;ygpzQ^Vx_e*q`q4+5}B&mFos;8swtxxBnst<;$`TYmcq0MT{=SqmXf(*=R zP~M7sZIB8E;Fq@c64~(6KF?&B9QY@$dJ)WxAF-;Q9SD%>>OjAx*hMg&?=&)zq+0Mp z%-gdAz0tZ==u0Yz4{ICbhi7qgBMo#F3M0W8d9kGe19z_Qea&U))WJW2VM6hfgS8Hw z8EKraxW2xX_XqnBN{9&fcJ!B<@VbnLWd>4Em9+3YQL*%0*S4V;pNQ&>^Mui_ge8z0 zFb%v+PE==yc$I9Pq8xJ<`f5CJ6|sycQY)Vm05hSVG|1w*6w8{9uA-Kne?4;lTI=We zC{-Q($W)n8Y9zL+i1?4y!YQi9nVU)8z&2wJ8e*xE)+w~|T17;&dIvzr%miV)#IM-| zHTjI}G+LoS!|eG*!Sc5NcIg)D#Pj>va(ZpjmHIb4zd1g13D5idMTUIoF~X{0o@o{+ z4$<^)9yk{bKAs6<<3?ktJ72Lm0fj;Ni$9fTLj}LD2qhqL`a&7ne1D+e1UqW#v^HVA zOOdBcZ24JX$8gPBgRbjWs!;^iT1pW??Z=T%K?BW!Jv*MPkAjW`e3{;;uZg|OazZnn4O;z|5fVpYHg0Jho)1~|lB`NqHH|LT^LDZBF#>;IuSeQAq0|sxX zR;r(ieDQr}c8ikg5>%BId-6>HiIFJkvgD z!eC$*lnS=uqSw8c_NxJ0_jj}w`CA)eB9`Q~(W#u_F8aI4oj3W{|NOt%^ZvhOU;bbG z+dU`+n6di)^}Bk%?AeRWEHTSACrcsNj6;J6sde_=WxuZ7+NGgF{lrz4od@GAkC*F; zmCb(j21=D#nRB18S?pWk=g9k#(gr6|BEyxKe<0YFJFr422@q3~)323DtXCLEF6E#s zY(^o1+K~9l_~Em^z8pCnwy8X`gTifGY1(F=X@_1-;{Wvv97@Cf9K?$Kzij&TSplQD6#m-CF69>`1UY~f#M`A?`+;#N zd6?oY<=v<=3-^sDER%-*+`aRakQSLN2XpM6did|}vD{BG{r&OV&tlD6EOyUwAQl)| zG&nhI{r3~N%0BW;sVsg<4)(7HG3L?4InuJhF35(~L|hu0En6QS=D! zg}3boi})a!2{EhVB?C3Y?x(f{aa>gLKd#@W-?h!Y?x~;aa3!XDyUwowtNR6MNvi-m zt>!l6U&&H9c~wN^o7hL*02j}Xk$rD@R2VSEWO(EttyEZGUO9cT+CarWAMXm4d_6i` z&^S0m?&I&X;g|c}w(&JJK+EO3rl7C>CEOdVA#`^*w-Fyds-(8I6%xh*% z6lh6C9v>V%?ebVzXa46O<7&i6G#157_msf6tg)8;#xZ%~ zcr`9t5zj~WZ&z1URW*v2aY$|}hn*~FHWrgslH8``t%f~)OPhq!|L$muOn`j48b{82 z-G)qGIx0Lq5Nn$?EBLpY*#;G0o)OG9V*87X^Q~Kue~}^>&HUCpB)aZ!PBNIM)_;q| zck0S@T?+|J-@w2E-38^9?rfkFb-zOfEz+lOll_+#I;t zPHO))`N(+R)lC)?`Iq;@AOa)qlOj6Sp*xGn7-NpOynr33PBP#=P}hl|<7Qh)y?d-= zjC{4TL#Od0X~(RNl{cUnva*%=kh?7pK57hk_jx@%k_k&0^n5w}rJm*bpy>4zf4c`9L3gdQKSYAIPm561g57Jj;$l zV>cGU&`+u)oC}-PEtP?5__mntO|>Cz0gs?u^N1}!xlZjsZ3so?KU65Qz0DK;llubl#u7pcH_1m69Y6zQ2|Hv$-UmzHx+YTj4KatDNbzO5y%|-RZSE%Xd7le zQ%ilJ+i_xS_t33Qo=`8#W~Z+pK%ETguDEM*l#L`9X`E`7 ze#6V^*j%S^vvu1(>Ye*0T;>BL*R|!X(QJ#WYRH^hh#J4|x#E_e zd%10R6wye=`R6ttBeJ)@yJJs>3#ILLmPqa$DU>i<+aoF_o|yX|*4o_NGiN+3;91L_ zP)Iqvv40hrWWRgSIm(=L3!ZJmIR!tQO~JXEP)cNXqsos%lSkh3nr&O03* zFu%B1wKF(#oS0~jmAH14A$SrQY&4}|=b9=2zqoC0W>zF5B(BfOjXx}emg*|#Cfgzu zH6{5~zW%ewH^h9zpSTmXaa=!01AhqF)V8 zKERF#1yAlweTYAlLI|F|JiJorwkLZKtfeJ_l1?f(fX$-Z&IZ}cu40;zWHmjC44fFHtHaIX7=0(3u>~&S1NF{4wDmgWiMvK z8~NSU_XFwQ-mmJL>G&$OE{;6>p8jPm*-(8`Fm*klG%(z0j{0No!Ul{%VAevZkYO|C zp*mK8Dx-{ifK*+@xojl>MtNXon9izVnUjaA+cqUx^UhOcsrHo5nl(QfO3ROqg0pXI zC(H8Q7L4I=Dr=(x5DMt@?qgM%JXHkvyLg6ZXL@aj~+v}sYWP;mtXmz7b*YFwC=2SV*^petix34KEXVywORT#^IOX1|_R`cl`fE$g%Ksy#`@?*$gR6LVZ?NL3cNmcv(Y3p z6gT7742g-@jd?P4=*udx(*l4Mj@KBB>Q|IH1)@g>WufhYBe8pHjge(-Js%@MB31UpsGJHg3NW3xN3@diib_V;@1E1qzBXwUeX(8O4*w-_m_!GRLb= zFe=Kyg?|PhF&GnRbgD5F%Byeb^MqWIKW$x8mqhQfh;PpMy-2&;>X3ESHpHu6=OQWG zKr!Eymt)(`F)Ims`ZXA3iAIGXWZooqK+h9Y7&@SvnM0lkvh92u%(__oh5d32P)SFw zkD=!TvayN5*p=$~vl%P&Oe}d?IsU*+1@2g{Neam5ij*sFHk_nEP0fWNpHawjcmM1& z79K+pi_hGr4OB+@(}9!08%fLPD;Qg(X*q=mAiKC^p2m4xfm|A*`n!$7JtsVBX7;HM zFO9uzQpnYeX^gD)aQo$yz-axPSswqcVPY+aG1T?|6PdZ{0*D{pJUg}7=0Rbo{-!GS zjJR);KMWBZ@8DO=nMxq-q)VyrD?VKtwTqUo{b1$O?ArZFD8FVNk)>~UUYUoBIQd+` zd<#Ea8ZFF>({f_s5KfR!f)~qOAEm@?zQ=(Qbn=oI)s}1@@P;4>n8-T>Q!8aI@1*WK z>ZEg&&Hk95FB&k~T>fT!cGT&BN|*?>DGEk6rPgTK@xsU`n>!(4@ajaQ)ktW(ot_j4jMoy3Y{hx{o$P~4zNL+1$+m9G7ArSM?t zl#DFQKCV&oEBFE?b>g0v*rk>)LtI_GNtBb{A!v7Md8$lMUw(l*)QKuY#6&RT+PBw1 zOuJpoQpv>ogc{kEJN6;>(Y0Y2EDSX{g`F_|gJ(xg=3(oC#kDr>{%YZE@@_?A=S-8o zU48*X!>9xP+kf9UKcm!9-eGyWEL_CJ`2LtucC)gqEFi6LsD}pGIPtLwhHhT0(i!P2 zz1Z%Vb-T>wBX637bMzzc>Khx6_p~ zS&dG~{TFwBx-p)a`JOIwt&{eXUof#CQQh%olHM?pQK+emMm;n zZLjE~s(KZxLpl)FD7g-3l_^c-jjhx)6?@Jp1YLGHRWC){V0C6@7o@sBb&7fD$|IH+ zOd^*-B&Y?blZ^*{77}{9y9|7T?y;T?3KCP&0YR=VNRGUf5~ zf&_gDe|;m`<)L`+Fd0ySwIvki4_<35n$_CW7P@mR1{RZ~#!y$X1Z0N$`@gdbjv z3fT!qsYl{!wL>v1+~Qsib1YGt0V2?}N{*z|p7#4rmtiV_GNJ|+TK7g(BV~aN%4qNo zl<~r(M*Efi%z@PBs4$|pW6^(l6Z4G}Tt-tm*`GtqLun@g4d1Qe#`YI9ap0ChrRB zgXNkrBgNG&^y#61w|ipBAxdJ{SBVrJj;Kh^v`whiw>y`p9`ySjehVG+x_tE1)y$7$ zJGRDI$k)-M;0=HQb=A zx=b5992avpd?DMRMcWKmE#`Jvh6UtZS(k02cdX;viINJxNOZ=QoMBM#QyR_N{}nKDuCxqtW(+X_6zM{m-K^@pZ- zg+ZciLiZS+Ef?7Xd{6P=EOYr3iH47Z?rxzm%8d8)RP^mAclXr|olpC=W=fWYi-jf= z;=CV}(%1WJzejJ7)$9*3yVhaT$}Bntq5~!M4(Jbpx%pwdk^?e6HFcEQT%JEt4OiS9 zt2^eHcGX7JdW7J?#g}efBFVk%W_?3pawR4~hd zi`tLn*YYYic}LoB-rzH<)8=l1+S+Kkj9(6oet*sJTGA~Ee6x1wS*>d+&-7DH5|pd^ z=l^voDV#VgVvEZ^+=04;b!T9?Yj6+a6cUs%-upkwsKrxEF1Wo2cue`aPT8@}n) z{5)+fVr*=zE#K_*m4gRXR`p#dxt1S&FK^h7C0Ot3UBnKqsRfLgtzXg?oRH2`G@WA+ ww@8Q8=82Z6N99~OA}5?YDSR_C&iU~8;#Xtc^bTwIDd6X-&JFE?D^^ec8$LO^&;S4c literal 0 HcmV?d00001 diff --git a/opencloud/charts/loki/scenarios/ingress-values.yaml b/opencloud/charts/loki/scenarios/ingress-values.yaml new file mode 100644 index 0000000..ff5ff1e --- /dev/null +++ b/opencloud/charts/loki/scenarios/ingress-values.yaml @@ -0,0 +1,30 @@ +--- +gateway: + ingress: + enabled: true + annotations: {} + hosts: + - host: gateway.loki.example.com + paths: + - path: / + pathType: Prefix +loki: + commonConfig: + replication_factor: 1 + useTestSchema: true + storage: + bucketNames: + chunks: chunks + ruler: ruler + admin: admin +read: + replicas: 1 +write: + replicas: 1 +backend: + replicas: 1 +monitoring: + lokiCanary: + enabled: false +test: + enabled: false diff --git a/opencloud/charts/loki/scenarios/legacy-monitoring-values.yaml b/opencloud/charts/loki/scenarios/legacy-monitoring-values.yaml new file mode 100644 index 0000000..ad520e5 --- /dev/null +++ b/opencloud/charts/loki/scenarios/legacy-monitoring-values.yaml @@ -0,0 +1,27 @@ +--- +loki: + commonConfig: + replication_factor: 1 + useTestSchema: true + storage: + bucketNames: + chunks: chunks + ruler: ruler + admin: admin +read: + replicas: 1 +write: + replicas: 1 +backend: + replicas: 1 +monitoring: + enabled: true + selfMonitoring: + enabled: true + grafanaAgent: + installOperator: true + serviceMonitor: + labels: + release: "prometheus" +test: + prometheusAddress: "http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local.:9090" diff --git a/opencloud/charts/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml b/opencloud/charts/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml new file mode 100644 index 0000000..28c6c3b --- /dev/null +++ b/opencloud/charts/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml @@ -0,0 +1,67 @@ +loki: + # -- Storage config. Providing this will automatically populate all necessary storage configs in the templated config. + storage: + # Loki requires a bucket for chunks and the ruler. GEL requires a third bucket for the admin API. + # Please provide these values if you are using object storage. + bucketNames: + chunks: aws-s3-chunks-bucket + ruler: aws-s3-ruler-bucket + admin: aws-s3-admin-bucket + type: s3 + s3: + region: eu-central-1 + # -- Check https://grafana.com/docs/loki/latest/configuration/#schema_config for more info on how to configure schemas + schemaConfig: + configs: + - from: "2023-09-19" + index: + period: 1d + prefix: tsdb_index_ + object_store: s3 + schema: v13 + store: tsdb +###################################################################################################################### +# +# Enterprise Loki Configs +# +###################################################################################################################### + +# -- Configuration for running Enterprise Loki +enterprise: + # Enable enterprise features, license must be provided + enabled: true + # -- Grafana Enterprise Logs license + license: + contents: "content of licence" + tokengen: + annotations: { + eks.amazonaws.com/role-arn: arn:aws:iam::2222222:role/test-role + } + # -- Configuration for `provisioner` target + provisioner: + # -- Additional annotations for the `provisioner` Job + annotations: { + eks.amazonaws.com/role-arn: arn:aws:iam::2222222:role/test-role + } +###################################################################################################################### +# +# Service Accounts and Kubernetes RBAC +# +###################################################################################################################### +serviceAccount: + # -- Annotations for the service account + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::2222222:role/test-role + +# Configuration for the write pod(s) +write: + persistence: + storageClass: gp2 +# -- Configuration for the read pod(s) +read: + persistence: + storageClass: gp2 +# -- Configuration for the backend pod(s) +backend: + persistence: + storageClass: gp2 diff --git a/opencloud/charts/loki/simple-scalable-values.yaml b/opencloud/charts/loki/simple-scalable-values.yaml new file mode 100644 index 0000000..78132b6 --- /dev/null +++ b/opencloud/charts/loki/simple-scalable-values.yaml @@ -0,0 +1,63 @@ +--- +loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + tracing: + enabled: true + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 4 + +#gateway: +# ingress: +# enabled: true +# hosts: +# - host: FIXME +# paths: +# - path: / +# pathType: Prefix + +deploymentMode: SimpleScalable + +backend: + replicas: 3 +read: + replicas: 3 +write: + replicas: 3 + +# Enable minio for storage +minio: + enabled: true + +# Zero out replica counts of other deployment modes +singleBinary: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/opencloud/charts/loki/single-binary-values.yaml b/opencloud/charts/loki/single-binary-values.yaml new file mode 100644 index 0000000..584f0fb --- /dev/null +++ b/opencloud/charts/loki/single-binary-values.yaml @@ -0,0 +1,79 @@ +--- +loki: + commonConfig: + replication_factor: 1 + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + tracing: + enabled: true + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 2 + +#gateway: +# ingress: +# enabled: true +# hosts: +# - host: FIXME +# paths: +# - path: / +# pathType: Prefix + +deploymentMode: SingleBinary +singleBinary: + replicas: 1 + resources: + limits: + cpu: 3 + memory: 4Gi + requests: + cpu: 2 + memory: 2Gi + extraEnv: + # Keep a little bit lower than memory limits + - name: GOMEMLIMIT + value: 3750MiB + +chunksCache: + # default is 500MB, with limited memory keep this smaller + writebackSizeLimit: 10MB + +# Enable minio for storage +minio: + enabled: true + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/opencloud/charts/loki/src/.yamllint.yaml b/opencloud/charts/loki/src/.yamllint.yaml new file mode 100644 index 0000000..19e5933 --- /dev/null +++ b/opencloud/charts/loki/src/.yamllint.yaml @@ -0,0 +1,4 @@ +--- +rules: + quoted-strings: + required: true diff --git a/opencloud/charts/loki/src/alerts.yaml.tpl b/opencloud/charts/loki/src/alerts.yaml.tpl new file mode 100644 index 0000000..0aa37b7 --- /dev/null +++ b/opencloud/charts/loki/src/alerts.yaml.tpl @@ -0,0 +1,78 @@ +--- +groups: + - name: "loki_alerts" + rules: +{{- if not (.Values.monitoring.rules.disabled.LokiRequestErrors | default false) }} + - alert: "LokiRequestErrors" + annotations: + message: | + {{`{{`}} $labels.job {{`}}`}} {{`{{`}} $labels.route {{`}}`}} is experiencing {{`{{`}} printf "%.2f" $value {{`}}`}}% errors. + expr: | + 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route) + / + sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route) + > 10 + for: "15m" + labels: + severity: "critical" +{{- if .Values.monitoring.rules.additionalRuleLabels }} +{{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }} +{{- end }} +{{- end }} +{{- if not (.Values.monitoring.rules.disabled.LokiRequestPanics | default false) }} + - alert: "LokiRequestPanics" + annotations: + message: | + {{`{{`}} $labels.job {{`}}`}} is experiencing {{`{{`}} printf "%.2f" $value {{`}}`}}% increase of panics. + expr: | + sum(increase(loki_panic_total[10m])) by (namespace, job) > 0 + labels: + severity: "critical" +{{- if .Values.monitoring.rules.additionalRuleLabels }} +{{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }} +{{- end }} +{{- end }} +{{- if not (.Values.monitoring.rules.disabled.LokiRequestLatency | default false) }} + - alert: "LokiRequestLatency" + annotations: + message: | + {{`{{`}} $labels.job {{`}}`}} {{`{{`}} $labels.route {{`}}`}} is experiencing {{`{{`}} printf "%.2f" $value {{`}}`}}s 99th percentile latency. + expr: | + namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*"} > 1 + for: "15m" + labels: + severity: "critical" +{{- if .Values.monitoring.rules.additionalRuleLabels }} +{{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }} +{{- end }} +{{- end }} +{{- if not (.Values.monitoring.rules.disabled.LokiTooManyCompactorsRunning | default false) }} + - alert: "LokiTooManyCompactorsRunning" + annotations: + message: | + {{`{{`}} $labels.cluster {{`}}`}} {{`{{`}} $labels.namespace {{`}}`}} has had {{`{{`}} printf "%.0f" $value {{`}}`}} compactors running for more than 5m. Only one compactor should run at a time. + expr: | + sum(loki_boltdb_shipper_compactor_running) by (cluster, namespace) > 1 + for: "5m" + labels: + severity: "warning" +{{- if .Values.monitoring.rules.additionalRuleLabels }} +{{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }} +{{- end }} +{{- end }} +{{- if not (.Values.monitoring.rules.disabled.LokiCanaryLatency | default false) }} + - name: "loki_canaries_alerts" + rules: + - alert: "LokiCanaryLatency" + annotations: + message: | + {{`{{`}} $labels.job {{`}}`}} is experiencing {{`{{`}} printf "%.2f" $value {{`}}`}}s 99th percentile latency. + expr: | + histogram_quantile(0.99, sum(rate(loki_canary_response_latency_seconds_bucket[5m])) by (le, namespace, job)) > 5 + for: "15m" + labels: + severity: "warning" +{{- if .Values.monitoring.rules.additionalRuleLabels }} +{{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/src/dashboards/loki-chunks.json b/opencloud/charts/loki/src/dashboards/loki-chunks.json new file mode 100644 index 0000000..bec1997 --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-chunks.json @@ -0,0 +1,1336 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunks per series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Active Series / Chunks", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Age", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Flush Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Log Entries Per Chunk", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Index Entries", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Index Entries Per Chunk", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Flush Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Queue Length", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Flush Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Flush Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunks Flushed/Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{reason}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Flush Reason", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Flush Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "heatmap": { }, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 11, + "legend": { + "show": true + }, + "span": 12, + "targets": [ + { + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "format": "heatmap", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Chunk Utilization", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "percentunit", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Utilization", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "heatmap": { }, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 12, + "legend": { + "show": true + }, + "span": 12, + "targets": [ + { + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Chunk Size Bytes", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "bytes", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Utilization", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p99", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p90", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p50", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Size Quantiles", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Utilization", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p50", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p99", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Duration hours (end-start)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Duration", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Chunks", + "uid": "chunks", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-deletion.json b/opencloud/charts/loki/src/dashboards/loki-deletion.json new file mode 100644 index 0000000..84bfee6 --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-deletion.json @@ -0,0 +1,632 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "none", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Number of Pending Requests", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "dtdurations", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Oldest Pending Request Age", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "received", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Delete Requests Received / Day", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "processed", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Delete Requests Processed / Day", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Churn", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "failures", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Failures in Loading Delete Requests / Hour", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Failures", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{user}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Lines Deleted / Sec", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Deleted lines", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Deletion", + "uid": "deletion", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-logs.json b/opencloud/charts/loki/src/dashboards/loki-logs.json new file mode 100644 index 0000000..0f113cf --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-logs.json @@ -0,0 +1,1073 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": 8, + "iteration": 1583185057230, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)", + "legendFormat": "{{quantile}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "gc duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "cpu", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "working set", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "tx", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 0 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "rx", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "restarts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", + "legendFormat": "{{level}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "bad words", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$logs", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "warn", + "color": "#FF780A" + }, + { + "alias": "error", + "color": "#E02F44" + }, + { + "alias": "info", + "color": "#56A64B" + }, + { + "alias": "debug", + "color": "#3274D9" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" [5m])) by (level)", + "intervalFactor": 3, + "legendFormat": "{{level}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Log Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$logs", + "gridPos": { + "h": 19, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 29, + "maxDataPoints": "", + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Logs", + "type": "logs" + } + ], + "refresh": "10s", + "rows": [ ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "logs", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "deployment", + "options": [ ], + "query": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)", + "refresh": 0, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", + "refresh": 0, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "container", + "options": [ ], + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", + "refresh": 0, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "", + "value": "" + }, + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "name": "level", + "options": [ + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + } + ], + "query": "debug,info,warn,error", + "refresh": 0, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "label": "LogQL Filter", + "name": "filter", + "query": "", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Logs", + "uid": "logs", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-mixin-recording-rules.json b/opencloud/charts/loki/src/dashboards/loki-mixin-recording-rules.json new file mode 100644 index 0000000..fe49ec7 --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-mixin-recording-rules.json @@ -0,0 +1,657 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ ], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": "${datasource}", + "enable": false, + "expr": "sum by (tenant) (changes(loki_ruler_wal_prometheus_tsdb_wal_truncations_total{tenant=~\"${tenant}\"}[$__rate_interval]))", + "iconColor": "red", + "name": "WAL Truncations", + "target": { + "queryType": "Azure Monitor", + "refId": "Anno" + }, + "titleFormat": "{{tenant}}" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1635347545534, + "links": [ ], + "liveNow": false, + "panels": [ + { + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.0-38205pre", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": false, + "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Appenders Not Ready", + "type": "stat" + }, + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 2, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Samples Appended to WAL per Second", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Series are unique combinations of labels", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 13, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Series Created per Second", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n (\n loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n or vector(0)\n )", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Write Behind", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Samples Sent per Second", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "WAL Disk Size", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0", + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Pending Samples", + "type": "timeseries" + } + ], + "schemaVersion": 31, + "style": "dark", + "tags": [ ], + "templating": { + "list": [ + { + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "datasource": "${datasource}", + "definition": "label_values(loki_ruler_wal_samples_appended_total, tenant)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": true, + "name": "tenant", + "options": [ ], + "query": { + "query": "label_values(loki_ruler_wal_samples_appended_total, tenant)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { }, + "timezone": "", + "title": "Recording Rules", + "uid": "2xKA_ZK7k", + "version": 9, + "weekStart": "" + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-operational.json b/opencloud/charts/loki/src/dashboards/loki-operational.json new file mode 100644 index 0000000..b69db23 --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-operational.json @@ -0,0 +1,6173 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": 68, + "iteration": 1588704280892, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [ ], + "targets": [ ], + "title": "Main", + "type": "row" + }, + { + "aliasColors": { + "5xx": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Queries/Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "5xx": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Pushes/Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Lines Per Tenant (top 10)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "MBs Per Tenant (Top 10)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", + "hide": false, + "interval": "", + "legendFormat": "{{container}}-{{pod}}", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Container Restarts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Push Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Distributor Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 6 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)", + "interval": "", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Distributor Success Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "hide": false, + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "hide": false, + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Ingester Latency Write", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 11 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)", + "interval": "", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Ingester Success Rate Write", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "legendFormat": "{{route}}-.99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "legendFormat": "{{route}}-.9", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "legendFormat": "{{route}}-.5", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Query Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".99-{{route}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".9-{{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".5-{{route}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Querier Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 16 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[5m])) by (route)", + "interval": "", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Querier Success Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".99-{{route}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".9-{{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "legendFormat": ".5-{{route}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Ingester Latency Read", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { } + }, + "overrides": [ ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 21 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)", + "interval": "", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Ingester Success Rate Read", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 110, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 112, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", + "interval": "", + "legendFormat": "{{ tenant }} - {{ reason }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Discarded Lines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [ ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 113, + "pageSize": null, + "panels": [ ], + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "tenant", + "thresholds": [ ], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "reason", + "thresholds": [ ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "right", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{ tenant }} - {{ reason }}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Discarded Lines Per Interval", + "transform": "table", + "type": "table-old" + } + ], + "targets": [ ], + "title": "Limits", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 23, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", + "intervalFactor": 3, + "legendFormat": "{{pod}}-{{container}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 28 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", + "instant": false, + "intervalFactor": 3, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$logs", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "{}", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Error Log Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$logs", + "gridPos": { + "h": 18, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 29, + "options": { + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "panels": [ ], + "targets": [ + { + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Success Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 35 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (ingester)", + "intervalFactor": 1, + "legendFormat": "{{ingester}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Append Failures By Ingester", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Bytes Received/Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 42 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Lines Received/Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Write Path", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 104, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 106, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", + "interval": "", + "legendFormat": "{{ tenant }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 108, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", + "interval": "", + "legendFormat": "{{ tenant }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Streams Created/Sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Streams", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 94, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "De-Dupe Ratio", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", + "interval": "", + "legendFormat": "Chunks", + "refId": "A" + }, + { + "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", + "interval": "", + "legendFormat": "De-Dupe Ratio", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Chunks Flushed/Sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "heatmap": { }, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 100, + "legend": { + "show": true + }, + "panels": [ ], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Size Bytes", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{ reason }}" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Chunk Flush Reason %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 39 + }, + "heatmap": { }, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 98, + "legend": { + "show": true + }, + "panels": [ ], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Utilization", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "percentunit", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "targets": [ ], + "title": "Chunks", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 64, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", + "intervalFactor": 3, + "legendFormat": "{{pod}}-{{container}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", + "instant": false, + "intervalFactor": 3, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$logs", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 3, + "w": 18, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "{}", + "color": "#F2495C" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Error Log Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$logs", + "gridPos": { + "h": 18, + "w": 18, + "x": 12, + "y": 35 + }, + "id": 66, + "options": { + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "panels": [ ], + "targets": [ + { + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 46 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[1m])) by (route)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Success Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Read Path", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 52, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 53, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "intervalFactor": 1, + "legendFormat": "{{container}}: .99-{{method}}-{{name}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "hide": false, + "legendFormat": "{{container}}: .9-{{method}}-{{name}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "hide": false, + "legendFormat": "{{container}}: .5-{{method}}-{{name}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 54, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", + "intervalFactor": 1, + "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Memcached", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 57, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 55, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 58, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Consul", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 43, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 41, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".9", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "MutateRows Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 9 + }, + "hiddenSeries": false, + "id": 46, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "interval": "", + "legendFormat": "90%", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "interval": "", + "legendFormat": "50%", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "ReadRows Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 44, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "interval": "", + "legendFormat": "90%", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "interval": "", + "legendFormat": "50%", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "GetTable Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 9 + }, + "hiddenSeries": false, + "id": 45, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".9", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "ListTables Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 47, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "MutateRows Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 16 + }, + "hiddenSeries": false, + "id": 50, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "ReadRows Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 48, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "GetTable Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 16 + }, + "hiddenSeries": false, + "id": 49, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "ListTables Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Big Table", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 60, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 61, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 62, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "GCS", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 76, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Failure Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 83, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Consumed Capacity Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 84, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Throttled Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Dropped Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 15 + }, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Query Pages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 9, + "x": 6, + "y": 15 + }, + "id": 87, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 15 + }, + "id": 88, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Dynamo", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 78, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 79, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 80, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "S3", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 78, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 79, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 80, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "Azure Blob", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 114, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 115, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "intervalFactor": 1, + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "hide": false, + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 116, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ ] + }, + "panels": [ ], + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "intervalFactor": 1, + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Status By Method", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "targets": [ ], + "title": "BoltDB Shipper", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "hide": 0, + "label": null, + "name": "logs", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Operational", + "uid": "operational", + "version": 0 + } diff --git a/opencloud/charts/loki/src/dashboards/loki-reads-resources.json b/opencloud/charts/loki/src/dashboards/loki-reads-resources.json new file mode 100644 index 0000000..6fa166f --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-reads-resources.json @@ -0,0 +1,964 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "gridPos": { }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Writes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "gridPos": { }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Reads", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-read.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{persistentvolumeclaim}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "loki_boltdb_shipper_query_readiness_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duration", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Query Readiness Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Read path", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Reads Resources", + "uid": "reads-resources", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-reads.json b/opencloud/charts/loki/src/dashboards/loki-reads.json new file mode 100644 index 0000000..6cae30f --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-reads.json @@ -0,0 +1,504 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job_route:loki_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}) by (route) / sum(job_route:loki_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}) by (route) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Read Path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "BoltDB Shipper", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Reads", + "uid": "reads", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-retention.json b/opencloud/charts/loki/src/dashboards/loki-retention.json new file mode 100644 index 0000000..7fc99ec --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-retention.json @@ -0,0 +1,1537 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resource Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeFromNow" + } + }, + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { }, + "textMode": "auto" + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"} * 1e3", + "format": "time_series", + "instant": true, + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last Compact and Mark Operation Success", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duration", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Compact and Mark Operations Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{success}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Compact and Mark Operations Per Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Compact and Mark", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{action}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Processed Tables Per Action", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{table}}-{{action}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Modified Tables", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{table}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Marks Creation Rate Per Table", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Per Table Marker", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "short", + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Marked Chunks (24h)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mark Table Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "short", + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Delete Chunks (24h)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Delete Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Sweeper", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster=~\"$cluster\", namespace=~\"$namespace\"} > 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "lag", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Sweeper Lag", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "count", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Marks Files to Process", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Delete Rate Per Status", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$logs", + "id": 17, + "span": 12, + "targets": [ + { + "expr": "{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}", + "refId": "A" + } + ], + "title": "Compactor Logs", + "type": "logs" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Logs", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "logs", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Retention", + "uid": "retention", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-writes-resources.json b/opencloud/charts/loki/src/dashboards/loki-writes-resources.json new file mode 100644 index 0000000..1b68bd3 --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-writes-resources.json @@ -0,0 +1,700 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "In-memory streams", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "gridPos": { }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Writes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "gridPos": { }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Reads", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{persistentvolumeclaim}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Write path", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Writes Resources", + "uid": "writes-resources", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/dashboards/loki-writes.json b/opencloud/charts/loki/src/dashboards/loki-writes.json new file mode 100644 index 0000000..ebaf33e --- /dev/null +++ b/opencloud/charts/loki/src/dashboards/loki-writes.json @@ -0,0 +1,504 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job:loki_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"}) / sum(job:loki_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Write Path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "BoltDB Shipper", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Writes", + "uid": "writes", + "version": 0 + } \ No newline at end of file diff --git a/opencloud/charts/loki/src/helm-test/Dockerfile b/opencloud/charts/loki/src/helm-test/Dockerfile new file mode 100644 index 0000000..9378d53 --- /dev/null +++ b/opencloud/charts/loki/src/helm-test/Dockerfile @@ -0,0 +1,13 @@ +ARG GO_VERSION=1.23 +FROM golang:${GO_VERSION} as build + +# build via Makefile target helm-test-image in root +# Makefile. Building from this directory will not be +# able to access source needed in rest of repo. +COPY . /src/loki +WORKDIR /src/loki +RUN make clean && make BUILD_IN_CONTAINER=false helm-test + +FROM gcr.io/distroless/static:debug +COPY --from=build /src/loki/production/helm/loki/src/helm-test/helm-test /usr/bin/helm-test +ENTRYPOINT [ "/usr/bin/helm-test" ] diff --git a/opencloud/charts/loki/src/helm-test/README.md b/opencloud/charts/loki/src/helm-test/README.md new file mode 100644 index 0000000..68c9bfd --- /dev/null +++ b/opencloud/charts/loki/src/helm-test/README.md @@ -0,0 +1,7 @@ +# Loki Helm Test + +This folder contains a collection of go tests that test if a Loki canary is running correctly. It's primary use it to test that the helm chart is working correctly by using metrics from the Loki canary. In the helm chart, the template for this test is only available if you are running both the Loki canary and have self monitoring enabled (as the Loki canary's logs need to be in Loki for it to work). However, the tests in this folder can be run against any running Loki canary using `go test`. + +## Instructions + +Run `go test .` from this directory, or use the Docker image published at `grafana/loki-helm-test`. diff --git a/opencloud/charts/loki/src/helm-test/canary_test.go b/opencloud/charts/loki/src/helm-test/canary_test.go new file mode 100644 index 0000000..002cae4 --- /dev/null +++ b/opencloud/charts/loki/src/helm-test/canary_test.go @@ -0,0 +1,178 @@ +//go:build helm_test +// +build helm_test + +package test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "testing" + "time" + + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + promConfig "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/textparse" + "github.com/stretchr/testify/require" +) + +type testResultFunc func(t *testing.T, ctx context.Context, metric string, test func(model.SampleValue) bool, msg string) error + +func TestCanary(t *testing.T) { + + var testResult testResultFunc + + // Default to directly querying a canary and looking for specific metrics. + testResult = testResultCanary + totalEntries := "loki_canary_entries_total" + totalEntriesMissing := "loki_canary_missing_entries_total" + + // For backwards compatibility and also for anyone who wants to validate with prometheus instead of querying + // a canary directly, if the CANARY_PROMETHEUS_ADDRESS is specified we will use prometheus to validate. + address := os.Getenv("CANARY_PROMETHEUS_ADDRESS") + if address != "" { + testResult = testResultPrometheus + // Use the sum function to aggregate the results from multiple canaries. + totalEntries = "sum(loki_canary_entries_total)" + totalEntriesMissing = "sum(loki_canary_missing_entries_total)" + } + + timeout := getEnv("CANARY_TEST_TIMEOUT", "1m") + timeoutDuration, err := time.ParseDuration(timeout) + require.NoError(t, err, "Failed to parse timeout. Please set CANARY_TEST_TIMEOUT to a valid duration.") + + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + + t.Cleanup(func() { + cancel() + }) + + t.Run("Canary should have entries", func(t *testing.T) { + eventually(t, func() error { + return testResult(t, ctx, totalEntries, func(v model.SampleValue) bool { + return v > 0 + }, fmt.Sprintf("Expected %s to be greater than 0", totalEntries)) + }, timeoutDuration, "Expected Loki Canary to have entries") + }) + + t.Run("Canary should not have missed any entries", func(t *testing.T) { + eventually(t, func() error { + return testResult(t, ctx, totalEntriesMissing, func(v model.SampleValue) bool { + return v == 0 + }, fmt.Sprintf("Expected %s to equal 0", totalEntriesMissing)) + }, timeoutDuration, "Expected Loki Canary to not have any missing entries") + }) +} + +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +func testResultPrometheus(t *testing.T, ctx context.Context, query string, test func(model.SampleValue) bool, msg string) error { + // TODO (ewelch): if we did a lot of these, we'd want to reuse the client but right now we only run a couple tests + client := newClient(t) + result, _, err := client.Query(ctx, query, time.Now()) + if err != nil { + return err + } + if v, ok := result.(model.Vector); ok { + for _, s := range v { + t.Logf("%s => %v\n", query, s.Value) + if !test(s.Value) { + return errors.New(msg) + } + } + return nil + } + + return fmt.Errorf("unexpected Prometheus result type: %v ", result.Type()) +} + +func newClient(t *testing.T) v1.API { + address := os.Getenv("CANARY_PROMETHEUS_ADDRESS") + require.NotEmpty(t, address, "CANARY_PROMETHEUS_ADDRESS must be set to a valid prometheus address") + + client, err := api.NewClient(api.Config{ + Address: address, + }) + require.NoError(t, err, "Failed to create Loki Canary client") + + return v1.NewAPI(client) +} + +func testResultCanary(t *testing.T, ctx context.Context, metric string, test func(model.SampleValue) bool, msg string) error { + address := os.Getenv("CANARY_SERVICE_ADDRESS") + require.NotEmpty(t, address, "CANARY_SERVICE_ADDRESS must be set to a valid kubernetes service for the Loki canaries") + + // TODO (ewelch): if we did a lot of these, we'd want to reuse the client but right now we only run a couple tests + client, err := promConfig.NewClientFromConfig(promConfig.HTTPClientConfig{}, "canary-test") + require.NoError(t, err, "Failed to create Prometheus client") + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) + require.NoError(t, err, "Failed to create request") + + rsp, err := client.Do(req) + if rsp != nil { + defer rsp.Body.Close() + } + require.NoError(t, err, "Failed to scrape metrics") + + body, err := io.ReadAll(rsp.Body) + require.NoError(t, err, "Failed to read response body") + + p, err := textparse.New(body, rsp.Header.Get("Content-Type"), true, nil) + require.NoError(t, err, "Failed to create Prometheus parser") + + for { + e, err := p.Next() + if err == io.EOF { + return errors.New("metric not found") + } + + if e != textparse.EntrySeries { + continue + } + + l := labels.Labels{} + p.Metric(&l) + + // Currently we aren't validating any labels, just the metric name, however this could be extended to do so. + name := l.Get(model.MetricNameLabel) + if name != metric { + continue + } + + _, _, val := p.Series() + t.Logf("%s => %v\n", metric, val) + + // Note: SampleValue has functions for comparing the equality of two floats which is + // why we convert this back to a SampleValue here for easier use intests. + if !test(model.SampleValue(val)) { + return errors.New(msg) + } + + // Returning here will only validate that one series was found matching the label name that met the condition + // it could be possible since we don't validate the rest of the labels that there is mulitple series + // but currently this meets the spirit of the test. + return nil + } +} + +func eventually(t *testing.T, test func() error, timeoutDuration time.Duration, msg string) { + require.Eventually(t, func() bool { + queryError := test() + if queryError != nil { + t.Logf("Query failed\n%+v\n", queryError) + } + return queryError == nil + }, timeoutDuration, 1*time.Second, msg) +} diff --git a/opencloud/charts/loki/src/helm-test/default.nix b/opencloud/charts/loki/src/helm-test/default.nix new file mode 100644 index 0000000..a129b23 --- /dev/null +++ b/opencloud/charts/loki/src/helm-test/default.nix @@ -0,0 +1,27 @@ +{ pkgs, lib, buildGoModule, dockerTools, rev }: +rec { + loki-helm-test = buildGoModule rec { + pname = "loki-helm-test"; + version = "0.1.0"; + + src = ./../../../../..; + vendorHash = null; + + buildPhase = '' + runHook preBuild + go test --tags=helm_test -c -o $out/bin/helm-test ./production/helm/loki/src/helm-test + runHook postBuild + ''; + + doCheck = false; + }; + + # by default, uses the nix hash as the tag, which can be retrieved with: + # basename "$(readlink result)" | cut -d - -f 1 + loki-helm-test-docker = dockerTools.buildImage { + name = "grafana/loki-helm-test"; + config = { + Entrypoint = [ "${loki-helm-test}/bin/helm-test" ]; + }; + }; +} diff --git a/opencloud/charts/loki/src/rules.yaml.tpl b/opencloud/charts/loki/src/rules.yaml.tpl new file mode 100644 index 0000000..840401d --- /dev/null +++ b/opencloud/charts/loki/src/rules.yaml.tpl @@ -0,0 +1,86 @@ +--- +groups: + - name: "loki_rules" + rules: + - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, job)) + record: job:loki_request_duration_seconds:99quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, job)) + record: job:loki_request_duration_seconds:50quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (job) / sum(rate(loki_request_duration_seconds_count[1m])) + by (job) + record: job:loki_request_duration_seconds:avg + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job) + record: job:loki_request_duration_seconds_bucket:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (job) + record: job:loki_request_duration_seconds_sum:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (job) + record: job:loki_request_duration_seconds_count:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, job, route)) + record: job_route:loki_request_duration_seconds:99quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, job, route)) + record: job_route:loki_request_duration_seconds:50quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (job, route) / sum(rate(loki_request_duration_seconds_count[1m])) + by (job, route) + record: job_route:loki_request_duration_seconds:avg + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route) + record: job_route:loki_request_duration_seconds_bucket:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (job, route) + record: job_route:loki_request_duration_seconds_sum:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (job, route) + record: job_route:loki_request_duration_seconds_count:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, namespace, job, route)) + record: namespace_job_route:loki_request_duration_seconds:99quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) + by (le, namespace, job, route)) + record: namespace_job_route:loki_request_duration_seconds:50quantile + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (namespace, job, route) + / sum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route) + record: namespace_job_route:loki_request_duration_seconds:avg + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, namespace, job, + route) + record: namespace_job_route:loki_request_duration_seconds_bucket:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_sum[1m])) by (namespace, job, route) + record: namespace_job_route:loki_request_duration_seconds_sum:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" + - expr: sum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route) + record: namespace_job_route:loki_request_duration_seconds_count:sum_rate + labels: + cluster: "{{ include "loki.clusterLabel" $ }}" diff --git a/opencloud/charts/loki/templates/NOTES.txt b/opencloud/charts/loki/templates/NOTES.txt new file mode 100644 index 0000000..622b1a8 --- /dev/null +++ b/opencloud/charts/loki/templates/NOTES.txt @@ -0,0 +1,184 @@ +*********************************************************************** + Welcome to Grafana Loki + Chart version: {{ .Chart.Version }} + Chart Name: {{ .Chart.Name }} + Loki version: {{ .Chart.AppVersion }} +*********************************************************************** + +** Please be patient while the chart is being deployed ** + +Tip: + + Watch the deployment status using the command: kubectl get pods -w --namespace {{ $.Release.Namespace }} + +If pods are taking too long to schedule make sure pod affinity can be fulfilled in the current cluster. + +*********************************************************************** +Installed components: +*********************************************************************** + +{{- if .Values.monitoring.selfMonitoring.enabled }} +* grafana-agent-operator +{{- end }} +{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }} +* loki +{{- else -}} +{{- if .Values.gateway.enabled }} +* gateway +{{- end }} +{{- if .Values.minio.enabled }} +* minio +{{- end }} +{{- if eq (include "loki.deployment.isScalable" .) "true" }} +* read +* write +{{- if not .Values.read.legacyReadTarget }} +* backend +{{- end }} +{{- else }} +* compactor +* index gateway +* query scheduler +* ruler +* distributor +* ingester +* querier +* query frontend +{{- end }} +{{- end }} + + +{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }} + +Loki has been deployed as a single binary. +This means a single pod is handling reads and writes. You can scale that pod vertically by adding more CPU and memory resources. + +{{- end }} + + +*********************************************************************** +Sending logs to Loki +*********************************************************************** + +{{- if .Values.gateway.enabled }} + +Loki has been configured with a gateway (nginx) to support reads and writes from a single component. + +{{- end }} + +You can send logs from inside the cluster using the cluster DNS: + +{{- if .Values.gateway.enabled }} + +http://{{ include "loki.gatewayFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local/loki/api/v1/push + +{{- else }} +{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }} + +http://{{ include "loki.singleBinaryFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/loki/api/v1/push + +{{- end}} +{{- if eq (include "loki.deployment.isScalable" .) "true" }} + +http://{{ include "loki.writeFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/loki/api/v1/push + +{{- end }} +{{- if eq (include "loki.deployment.isDistributed" .) "true" }} + +http://{{ include "loki.distributorFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:3100/loki/api/v1/push + +{{- end }} +{{- end }} + +You can test to send data from outside the cluster by port-forwarding the gateway to your local machine: +{{- if .Values.gateway.enabled }} + + kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.gatewayFullname" . }} 3100:{{ .Values.gateway.service.port }} & + +{{- else }} +{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }} + + kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.singleBinaryFullname" . }} 3100:{{ .Values.loki.server.http_listen_port }} & + +{{- end}} +{{- if eq (include "loki.deployment.isScalable" .) "true" }} + + kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.writeFullname" . }} 3100:{{ .Values.loki.server.http_listen_port }} & + +{{- end }} +{{- if eq (include "loki.deployment.isDistributed" .) "true" }} + + kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.distributorFullname" . }} 3100:3100 & + +{{- end }} +{{- end }} + +And then using http://127.0.0.1:3100/loki/api/v1/push URL as shown below: + +``` +curl -H "Content-Type: application/json" -XPOST -s "http://127.0.0.1:3100/loki/api/v1/push" \ +--data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}" +{{- if .Values.loki.auth_enabled }} \ +-H X-Scope-OrgId:foo +{{- end}} +``` + +Then verify that Loki did received the data using the following command: + +``` +curl "http://127.0.0.1:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' {{- if .Values.loki.auth_enabled }} -H X-Scope-OrgId:foo {{- end}} | jq .data.result +``` + +*********************************************************************** +Connecting Grafana to Loki +*********************************************************************** + +If Grafana operates within the cluster, you'll set up a new Loki datasource by utilizing the following URL: + +{{- if .Values.gateway.enabled }} + +http://{{ include "loki.gatewayFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local/ + +{{- else }} +{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }} + +http://{{ include "loki.singleBinaryFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/ + +{{- end}} +{{- if eq (include "loki.deployment.isScalable" .) "true" }} + +http://{{ include "loki.readFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/ + +{{- end }} +{{- if eq (include "loki.deployment.isDistributed" .) "true" }} + +http://{{ include "loki.queryFrontendFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:3100/ + +{{- end }} +{{- end }} + + + +{{- if .Values.loki.auth_enabled }} + +*********************************************************************** +Multi-tenancy +*********************************************************************** + +Loki is configured with auth enabled (multi-tenancy) and expects tenant headers (`X-Scope-OrgID`) to be set for all API calls. + +You must configure Grafana's Loki datasource using the `HTTP Headers` section with the `X-Scope-OrgID` to target a specific tenant. +For each tenant, you can create a different datasource. + +The agent of your choice must also be configured to propagate this header. +For example, when using Promtail you can use the `tenant` stage. https://grafana.com/docs/loki/latest/send-data/promtail/stages/tenant/ + +When not provided with the `X-Scope-OrgID` while auth is enabled, Loki will reject reads and writes with a 404 status code `no org id`. + +You can also use a reverse proxy, to automatically add the `X-Scope-OrgID` header as suggested by https://grafana.com/docs/loki/latest/operations/authentication/ + +For more information, read our documentation about multi-tenancy: https://grafana.com/docs/loki/latest/operations/multi-tenancy/ + +> When using curl you can pass `X-Scope-OrgId` header using `-H X-Scope-OrgId:foo` option, where foo can be replaced with the tenant of your choice. + +{{- end }} diff --git a/opencloud/charts/loki/templates/_helpers.tpl b/opencloud/charts/loki/templates/_helpers.tpl new file mode 100644 index 0000000..acc6690 --- /dev/null +++ b/opencloud/charts/loki/templates/_helpers.tpl @@ -0,0 +1,1132 @@ +{{/* +Enforce valid label value. +See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set +*/}} +{{- define "loki.validLabelValue" -}} +{{- (regexReplaceAllLiteral "[^a-zA-Z0-9._-]" . "-") | trunc 63 | trimSuffix "-" | trimSuffix "_" | trimSuffix "." }} +{{- end }} + +{{/* +Expand the name of the chart. +*/}} +{{- define "loki.name" -}} +{{- $default := ternary "enterprise-logs" "loki" .Values.enterprise.enabled }} +{{- coalesce .Values.nameOverride $default | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +singleBinary fullname +*/}} +{{- define "loki.singleBinaryFullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Resource name template +Params: + ctx = . context + component = component name (optional) + rolloutZoneName = rollout zone name (optional) +*/}} +{{- define "loki.resourceName" -}} +{{- $resourceName := include "loki.fullname" .ctx -}} +{{- if .component -}}{{- $resourceName = printf "%s-%s" $resourceName .component -}}{{- end -}} +{{- if and (not .component) .rolloutZoneName -}}{{- printf "Component name cannot be empty if rolloutZoneName (%s) is set" .rolloutZoneName | fail -}}{{- end -}} +{{- if .rolloutZoneName -}}{{- $resourceName = printf "%s-%s" $resourceName .rolloutZoneName -}}{{- end -}} +{{- if gt (len $resourceName) 253 -}}{{- printf "Resource name (%s) exceeds kubernetes limit of 253 character. To fix: shorten release name if this will be a fresh install or shorten zone names (e.g. \"a\" instead of \"zone-a\") if using zone-awareness." $resourceName | fail -}}{{- end -}} +{{- $resourceName -}} +{{- end -}} + +{{/* +Return if deployment mode is simple scalable +*/}} +{{- define "loki.deployment.isScalable" -}} + {{- and (eq (include "loki.isUsingObjectStorage" . ) "true") (or (eq .Values.deploymentMode "SingleBinary<->SimpleScalable") (eq .Values.deploymentMode "SimpleScalable") (eq .Values.deploymentMode "SimpleScalable<->Distributed")) }} +{{- end -}} + +{{/* +Return if deployment mode is single binary +*/}} +{{- define "loki.deployment.isSingleBinary" -}} + {{- or (eq .Values.deploymentMode "SingleBinary") (eq .Values.deploymentMode "SingleBinary<->SimpleScalable") }} +{{- end -}} + +{{/* +Return if deployment mode is distributed +*/}} +{{- define "loki.deployment.isDistributed" -}} + {{- and (eq (include "loki.isUsingObjectStorage" . ) "true") (or (eq .Values.deploymentMode "Distributed") (eq .Values.deploymentMode "SimpleScalable<->Distributed")) }} +{{- end -}} + + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "loki.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := include "loki.name" . }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Cluster label for rules and alerts. +*/}} +{{- define "loki.clusterLabel" -}} +{{- if .Values.clusterLabelOverride }} +{{- .Values.clusterLabelOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := include "loki.name" . }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Create a default storage config that uses filesystem storage +This is required for CI, but Loki will not be queryable with this default +applied, thus it is encouraged that users override this. +*/}} +{{- define "loki.storageConfig" -}} +{{- if .Values.loki.storageConfig -}} +{{- .Values.loki.storageConfig | toYaml | nindent 4 -}} +{{- else }} +{{- .Values.loki.defaultStorageConfig | toYaml | nindent 4 }} +{{- end}} +{{- end}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "loki.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "loki.labels" -}} +helm.sh/chart: {{ include "loki.chart" . }} +{{ include "loki.selectorLabels" . }} +{{- if or (.Chart.AppVersion) (.Values.loki.image.tag) }} +app.kubernetes.io/version: {{ include "loki.validLabelValue" (.Values.loki.image.tag | default .Chart.AppVersion) | quote }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "loki.selectorLabels" -}} +app.kubernetes.io/name: {{ include "loki.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "loki.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "loki.name" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Base template for building docker image reference +*/}} +{{- define "loki.baseImage" }} +{{- $registry := .global.registry | default .service.registry | default "" -}} +{{- $repository := .service.repository | default "" -}} +{{- $ref := ternary (printf ":%s" (.service.tag | default .defaultVersion | toString)) (printf "@%s" .service.digest) (empty .service.digest) -}} +{{- if and $registry $repository -}} + {{- printf "%s/%s%s" $registry $repository $ref -}} +{{- else -}} + {{- printf "%s%s%s" $registry $repository $ref -}} +{{- end -}} +{{- end -}} + +{{/* +Docker image name for Loki +*/}} +{{- define "loki.lokiImage" -}} +{{- $dict := dict "service" .Values.loki.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + +{{/* +Docker image name for enterprise logs +*/}} +{{- define "loki.enterpriseImage" -}} +{{- $dict := dict "service" .Values.enterprise.image "global" .Values.global.image "defaultVersion" .Values.enterprise.version -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + +{{/* +Docker image name +*/}} +{{- define "loki.image" -}} +{{- if .Values.enterprise.enabled -}}{{- include "loki.enterpriseImage" . -}}{{- else -}}{{- include "loki.lokiImage" . -}}{{- end -}} +{{- end -}} + +{{/* +Docker image name for kubectl container +*/}} +{{- define "loki.kubectlImage" -}} +{{- $dict := dict "service" .Values.kubectlImage "global" .Values.global.image "defaultVersion" "latest" -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + +{{/* +Generated storage config for loki common config +*/}} +{{- define "loki.commonStorageConfig" -}} +{{- if .Values.minio.enabled -}} +s3: + endpoint: {{ include "loki.minio" $ }} + bucketnames: chunks + secret_access_key: {{ $.Values.minio.rootPassword }} + access_key_id: {{ $.Values.minio.rootUser }} + s3forcepathstyle: true + insecure: true +{{- else if eq .Values.loki.storage.type "s3" -}} +{{- with .Values.loki.storage.s3 }} +s3: + {{- with .s3 }} + s3: {{ . }} + {{- end }} + {{- with .endpoint }} + endpoint: {{ . }} + {{- end }} + {{- with .region }} + region: {{ . }} + {{- end}} + bucketnames: {{ $.Values.loki.storage.bucketNames.chunks }} + {{- with .secretAccessKey }} + secret_access_key: {{ . }} + {{- end }} + {{- with .accessKeyId }} + access_key_id: {{ . }} + {{- end }} + {{- with .signatureVersion }} + signature_version: {{ . }} + {{- end }} + s3forcepathstyle: {{ .s3ForcePathStyle }} + insecure: {{ .insecure }} + {{- with .disable_dualstack }} + disable_dualstack: {{ . }} + {{- end }} + {{- with .http_config}} + http_config: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .backoff_config}} + backoff_config: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .sse }} + sse: +{{ toYaml . | indent 4 }} + {{- end }} +{{- end -}} + +{{- else if eq .Values.loki.storage.type "gcs" -}} +{{- with .Values.loki.storage.gcs }} +gcs: + bucket_name: {{ $.Values.loki.storage.bucketNames.chunks }} + chunk_buffer_size: {{ .chunkBufferSize }} + request_timeout: {{ .requestTimeout }} + enable_http2: {{ .enableHttp2 }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "azure" -}} +{{- with .Values.loki.storage.azure }} +azure: + account_name: {{ .accountName }} + {{- with .accountKey }} + account_key: {{ . }} + {{- end }} + {{- with .connectionString }} + connection_string: {{ . }} + {{- end }} + container_name: {{ $.Values.loki.storage.bucketNames.chunks }} + use_managed_identity: {{ .useManagedIdentity }} + use_federated_token: {{ .useFederatedToken }} + {{- with .userAssignedId }} + user_assigned_id: {{ . }} + {{- end }} + {{- with .requestTimeout }} + request_timeout: {{ . }} + {{- end }} + {{- with .endpointSuffix }} + endpoint_suffix: {{ . }} + {{- end }} + {{- with .chunkDelimiter }} + chunk_delimiter: {{ . }} + {{- end }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "alibabacloud" -}} +{{- with .Values.loki.storage.alibabacloud }} +alibabacloud: + bucket: {{ $.Values.loki.storage.bucketNames.chunks }} + endpoint: {{ .endpoint }} + access_key_id: {{ .accessKeyId }} + secret_access_key: {{ .secretAccessKey }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "swift" -}} +{{- with .Values.loki.storage.swift }} +swift: +{{ toYaml . | indent 2 }} +{{- end -}} +{{- else -}} +{{- with .Values.loki.storage.filesystem }} +filesystem: + chunks_directory: {{ .chunks_directory }} + rules_directory: {{ .rules_directory }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Storage config for ruler +*/}} +{{- define "loki.rulerStorageConfig" -}} +{{- if .Values.minio.enabled -}} +type: "s3" +s3: + bucketnames: ruler +{{- else if eq .Values.loki.storage.type "s3" -}} +{{- with .Values.loki.storage.s3 }} +type: "s3" +s3: + {{- with .s3 }} + s3: {{ . }} + {{- end }} + {{- with .endpoint }} + endpoint: {{ . }} + {{- end }} + {{- with .region }} + region: {{ . }} + {{- end}} + bucketnames: {{ $.Values.loki.storage.bucketNames.ruler }} + {{- with .secretAccessKey }} + secret_access_key: {{ . }} + {{- end }} + {{- with .accessKeyId }} + access_key_id: {{ . }} + {{- end }} + s3forcepathstyle: {{ .s3ForcePathStyle }} + insecure: {{ .insecure }} + {{- with .http_config }} + http_config: {{ toYaml . | nindent 6 }} + {{- end }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "gcs" -}} +{{- with .Values.loki.storage.gcs }} +type: "gcs" +gcs: + bucket_name: {{ $.Values.loki.storage.bucketNames.ruler }} + chunk_buffer_size: {{ .chunkBufferSize }} + request_timeout: {{ .requestTimeout }} + enable_http2: {{ .enableHttp2 }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "azure" -}} +{{- with .Values.loki.storage.azure }} +type: "azure" +azure: + account_name: {{ .accountName }} + {{- with .accountKey }} + account_key: {{ . }} + {{- end }} + {{- with .connectionString }} + connection_string: {{ . }} + {{- end }} + container_name: {{ $.Values.loki.storage.bucketNames.ruler }} + use_managed_identity: {{ .useManagedIdentity }} + use_federated_token: {{ .useFederatedToken }} + {{- with .userAssignedId }} + user_assigned_id: {{ . }} + {{- end }} + {{- with .requestTimeout }} + request_timeout: {{ . }} + {{- end }} + {{- with .endpointSuffix }} + endpoint_suffix: {{ . }} + {{- end }} +{{- end -}} +{{- else if eq .Values.loki.storage.type "swift" -}} +{{- with .Values.loki.storage.swift }} +swift: + {{- with .auth_version }} + auth_version: {{ . }} + {{- end }} + auth_url: {{ .auth_url }} + {{- with .internal }} + internal: {{ . }} + {{- end }} + username: {{ .username }} + user_domain_name: {{ .user_domain_name }} + {{- with .user_domain_id }} + user_domain_id: {{ . }} + {{- end }} + {{- with .user_id }} + user_id: {{ . }} + {{- end }} + password: {{ .password }} + {{- with .domain_id }} + domain_id: {{ . }} + {{- end }} + domain_name: {{ .domain_name }} + project_id: {{ .project_id }} + project_name: {{ .project_name }} + project_domain_id: {{ .project_domain_id }} + project_domain_name: {{ .project_domain_name }} + region_name: {{ .region_name }} + container_name: {{ .container_name }} + max_retries: {{ .max_retries | default 3 }} + connect_timeout: {{ .connect_timeout | default "10s" }} + request_timeout: {{ .request_timeout | default "5s" }} +{{- end -}} +{{- else }} +type: "local" +{{- end -}} +{{- end -}} + +{{/* Loki ruler config */}} +{{- define "loki.rulerConfig" }} +ruler: + storage: + {{- include "loki.rulerStorageConfig" . | nindent 4}} +{{- if (not (empty .Values.loki.rulerConfig)) }} +{{- toYaml .Values.loki.rulerConfig | nindent 2}} +{{- end }} +{{- end }} + +{{/* Enterprise Logs Admin API storage config */}} +{{- define "enterprise-logs.adminAPIStorageConfig" }} +storage: + {{- if .Values.minio.enabled }} + backend: "s3" + s3: + bucket_name: admin + {{- else if eq .Values.loki.storage.type "s3" -}} + {{- with .Values.loki.storage.s3 }} + backend: "s3" + s3: + bucket_name: {{ $.Values.loki.storage.bucketNames.admin }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "gcs" -}} + {{- with .Values.loki.storage.gcs }} + backend: "gcs" + gcs: + bucket_name: {{ $.Values.loki.storage.bucketNames.admin }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "azure" -}} + {{- with .Values.loki.storage.azure }} + backend: "azure" + azure: + account_name: {{ .accountName }} + {{- with .accountKey }} + account_key: {{ . }} + {{- end }} + {{- with .connectionString }} + connection_string: {{ . }} + {{- end }} + container_name: {{ $.Values.loki.storage.bucketNames.admin }} + {{- with .endpointSuffix }} + endpoint_suffix: {{ . }} + {{- end }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "swift" -}} + {{- with .Values.loki.storage.swift }} + backend: "swift" + swift: + {{- with .auth_version }} + auth_version: {{ . }} + {{- end }} + auth_url: {{ .auth_url }} + {{- with .internal }} + internal: {{ . }} + {{- end }} + username: {{ .username }} + user_domain_name: {{ .user_domain_name }} + {{- with .user_domain_id }} + user_domain_id: {{ . }} + {{- end }} + {{- with .user_id }} + user_id: {{ . }} + {{- end }} + password: {{ .password }} + {{- with .domain_id }} + domain_id: {{ . }} + {{- end }} + domain_name: {{ .domain_name }} + project_id: {{ .project_id }} + project_name: {{ .project_name }} + project_domain_id: {{ .project_domain_id }} + project_domain_name: {{ .project_domain_name }} + region_name: {{ .region_name }} + container_name: {{ .container_name }} + max_retries: {{ .max_retries | default 3 }} + connect_timeout: {{ .connect_timeout | default "10s" }} + request_timeout: {{ .request_timeout | default "5s" }} + {{- end -}} + {{- else }} + backend: "filesystem" + filesystem: + dir: {{ .Values.loki.storage.filesystem.admin_api_directory }} + {{- end -}} +{{- end }} + +{{/* +Calculate the config from structured and unstructured text input +*/}} +{{- define "loki.calculatedConfig" -}} +{{ tpl (mergeOverwrite (tpl .Values.loki.config . | fromYaml) .Values.loki.structuredConfig | toYaml) . }} +{{- end }} + +{{/* +The volume to mount for loki configuration +*/}} +{{- define "loki.configVolume" -}} +{{- if eq .Values.loki.configStorageType "Secret" -}} +secret: + secretName: {{ tpl .Values.loki.configObjectName . }} +{{- else -}} +configMap: + name: {{ tpl .Values.loki.configObjectName . }} + items: + - key: "config.yaml" + path: "config.yaml" +{{- end -}} +{{- end -}} + +{{/* +Memcached Docker image +*/}} +{{- define "loki.memcachedImage" -}} +{{- $dict := dict "service" .Values.memcached.image "global" .Values.global.image -}} +{{- include "loki.image" $dict -}} +{{- end }} + +{{/* +Memcached Exporter Docker image +*/}} +{{- define "loki.memcachedExporterImage" -}} +{{- $dict := dict "service" .Values.memcachedExporter.image "global" .Values.global.image -}} +{{- include "loki.image" $dict -}} +{{- end }} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "loki.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "loki.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "loki.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "loki.ingress.isStable" -}} + {{- eq (include "loki.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "loki.ingress.supportsIngressClassName" -}} + {{- or (eq (include "loki.ingress.isStable" .) "true") (and (eq (include "loki.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "loki.kubeVersion" .))) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "loki.ingress.supportsPathType" -}} + {{- or (eq (include "loki.ingress.isStable" .) "true") (and (eq (include "loki.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "loki.kubeVersion" .))) -}} +{{- end -}} + +{{/* +Generate list of ingress service paths based on deployment type +*/}} +{{- define "loki.ingress.servicePaths" -}} +{{- if (eq (include "loki.deployment.isSingleBinary" .) "true") -}} +{{- include "loki.ingress.singleBinaryServicePaths" . }} +{{- else if (eq (include "loki.deployment.isDistributed" .) "true") -}} +{{- include "loki.ingress.distributedServicePaths" . }} +{{- else if and (eq (include "loki.deployment.isScalable" .) "true") (not .Values.read.legacyReadTarget ) -}} +{{- include "loki.ingress.scalableServicePaths" . }} +{{- else -}} +{{- include "loki.ingress.legacyScalableServicePaths" . }} +{{- end -}} +{{- end -}} + + +{{/* +Ingress service paths for distributed deployment +*/}} +{{- define "loki.ingress.distributedServicePaths" -}} +{{- $distributorServiceName := include "loki.distributorFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $distributorServiceName "paths" .Values.ingress.paths.distributor )}} +{{- $queryFrontendServiceName := include "loki.queryFrontendFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $queryFrontendServiceName "paths" .Values.ingress.paths.queryFrontend )}} +{{- $rulerServiceName := include "loki.rulerFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $rulerServiceName "paths" .Values.ingress.paths.ruler)}} +{{- end -}} + +{{/* +Ingress service paths for legacy simple scalable deployment when backend components were part of read component. +*/}} +{{- define "loki.ingress.scalableServicePaths" -}} +{{- $readServiceName := include "loki.readFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $readServiceName "paths" .Values.ingress.paths.queryFrontend )}} +{{- $writeServiceName := include "loki.writeFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $writeServiceName "paths" .Values.ingress.paths.distributor )}} +{{- $backendServiceName := include "loki.backendFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $backendServiceName "paths" .Values.ingress.paths.ruler )}} +{{- end -}} + +{{/* +Ingress service paths for legacy simple scalable deployment +*/}} +{{- define "loki.ingress.legacyScalableServicePaths" -}} +{{- $readServiceName := include "loki.readFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $readServiceName "paths" .Values.ingress.paths.queryFrontend )}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $readServiceName "paths" .Values.ingress.paths.ruler )}} +{{- $writeServiceName := include "loki.writeFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $writeServiceName "paths" .Values.ingress.paths.distributor )}} +{{- end -}} + +{{/* +Ingress service paths for single binary deployment +*/}} +{{- define "loki.ingress.singleBinaryServicePaths" -}} +{{- $serviceName := include "loki.singleBinaryFullname" . }} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $serviceName "paths" .Values.ingress.paths.distributor )}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $serviceName "paths" .Values.ingress.paths.queryFrontend )}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "serviceName" $serviceName "paths" .Values.ingress.paths.ruler )}} +{{- end -}} + +{{/* +Ingress service path helper function +Params: + ctx = . context + serviceName = fully qualified k8s service name + paths = list of url paths to allow ingress for +*/}} +{{- define "loki.ingress.servicePath" -}} +{{- $ingressApiIsStable := eq (include "loki.ingress.isStable" .ctx) "true" -}} +{{- $ingressSupportsPathType := eq (include "loki.ingress.supportsPathType" .ctx) "true" -}} +{{- range .paths }} +- path: {{ . }} + {{- if $ingressSupportsPathType }} + pathType: Prefix + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $.serviceName }} + port: + number: {{ $.ctx.Values.loki.server.http_listen_port }} + {{- else }} + serviceName: {{ $.serviceName }} + servicePort: {{ $.ctx.Values.loki.server.http_listen_port }} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the service endpoint including port for MinIO. +*/}} +{{- define "loki.minio" -}} +{{- if .Values.minio.enabled -}} +{{- .Values.minio.address | default (printf "%s-%s.%s.svc:%s" .Release.Name "minio" .Release.Namespace (.Values.minio.service.port | toString)) -}} +{{- end -}} +{{- end -}} + +{{/* Determine if deployment is using object storage */}} +{{- define "loki.isUsingObjectStorage" -}} +{{- or (eq .Values.loki.storage.type "gcs") (eq .Values.loki.storage.type "s3") (eq .Values.loki.storage.type "azure") (eq .Values.loki.storage.type "swift") (eq .Values.loki.storage.type "alibabacloud") -}} +{{- end -}} + +{{/* Configure the correct name for the memberlist service */}} +{{- define "loki.memberlist" -}} +{{ include "loki.name" . }}-memberlist +{{- end -}} + +{{/* Determine the public host for the Loki cluster */}} +{{- define "loki.host" -}} +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- $url := printf "%s.%s.svc.%s.:%s" (include "loki.gatewayFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.gateway.service.port | toString) }} +{{- if and $isSingleBinary (not .Values.gateway.enabled) }} + {{- $url = printf "%s.%s.svc.%s.:%s" (include "loki.singleBinaryFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} +{{- end }} +{{- printf "%s" $url -}} +{{- end -}} + +{{/* Determine the public endpoint for the Loki cluster */}} +{{- define "loki.address" -}} +{{- printf "http://%s" (include "loki.host" . ) -}} +{{- end -}} + +{{/* Name of the cluster */}} +{{- define "loki.clusterName" -}} +{{- $name := .Values.enterprise.cluster_name | default .Release.Name }} +{{- printf "%s" $name -}} +{{- end -}} + +{{/* Name of kubernetes secret to persist GEL admin token to */}} +{{- define "enterprise-logs.adminTokenSecret" }} +{{- .Values.enterprise.adminToken.secret | default (printf "%s-admin-token" (include "loki.name" . )) -}} +{{- end -}} + +{{/* Prefix for provisioned secrets created for each provisioned tenant */}} +{{- define "enterprise-logs.provisionedSecretPrefix" }} +{{- .Values.enterprise.provisioner.provisionedSecretPrefix | default (printf "%s-provisioned" (include "loki.name" . )) -}} +{{- end -}} + +{{/* Name of kubernetes secret to persist canary credentials in */}} +{{- define "enterprise-logs.selfMonitoringTenantSecret" }} +{{- .Values.enterprise.canarySecret | default (printf "%s-%s" (include "enterprise-logs.provisionedSecretPrefix" . ) .Values.monitoring.selfMonitoring.tenant.name) -}} +{{- end -}} + +{{/* Snippet for the nginx file used by gateway */}} +{{- define "loki.nginxFile" }} +worker_processes 5; ## Default: 1 +error_log /dev/stderr; +pid /tmp/nginx.pid; +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; ## Default: 1024 +} + +http { + client_body_temp_path /tmp/client_temp; + proxy_temp_path /tmp/proxy_temp_path; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + client_max_body_size {{ .Values.gateway.nginxConfig.clientMaxBodySize }}; + + proxy_read_timeout 600; ## 10 minutes + proxy_send_timeout 600; + proxy_connect_timeout 600; + + proxy_http_version 1.1; + + default_type application/octet-stream; + log_format {{ .Values.gateway.nginxConfig.logFormat }} + + {{- if .Values.gateway.verboseLogging }} + access_log /dev/stderr main; + {{- else }} + + map $status $loggable { + ~^[23] 0; + default 1; + } + access_log /dev/stderr main if=$loggable; + {{- end }} + + sendfile on; + tcp_nopush on; + {{- if .Values.gateway.nginxConfig.resolver }} + resolver {{ .Values.gateway.nginxConfig.resolver }}; + {{- else }} + resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}.; + {{- end }} + + {{- with .Values.gateway.nginxConfig.httpSnippet }} + {{- tpl . $ | nindent 2 }} + {{- end }} + + server { + {{- if (.Values.gateway.nginxConfig.ssl) }} + listen 8080 ssl; + {{- if .Values.gateway.nginxConfig.enableIPv6 }} + listen [::]:8080 ssl; + {{- end }} + {{- else }} + listen 8080; + {{- if .Values.gateway.nginxConfig.enableIPv6 }} + listen [::]:8080; + {{- end }} + {{- end }} + + {{- if .Values.gateway.basicAuth.enabled }} + auth_basic "Loki"; + auth_basic_user_file /etc/nginx/secrets/.htpasswd; + {{- end }} + + location = / { + return 200 'OK'; + auth_basic off; + } + + ######################################################## + # Configure backend targets + + {{- $backendHost := include "loki.backendFullname" .}} + {{- $readHost := include "loki.readFullname" .}} + {{- $writeHost := include "loki.writeFullname" .}} + + {{- if .Values.read.legacyReadTarget }} + {{- $backendHost = include "loki.readFullname" . }} + {{- end }} + + {{- $httpSchema := .Values.gateway.nginxConfig.schema }} + + {{- $writeUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $writeHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $readUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $readHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $backendUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $backendHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + + {{- if .Values.gateway.nginxConfig.customWriteUrl }} + {{- $writeUrl = .Values.gateway.nginxConfig.customWriteUrl }} + {{- end }} + {{- if .Values.gateway.nginxConfig.customReadUrl }} + {{- $readUrl = .Values.gateway.nginxConfig.customReadUrl }} + {{- end }} + {{- if .Values.gateway.nginxConfig.customBackendUrl }} + {{- $backendUrl = .Values.gateway.nginxConfig.customBackendUrl }} + {{- end }} + + {{- $singleBinaryHost := include "loki.singleBinaryFullname" . }} + {{- $singleBinaryUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $singleBinaryHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + + {{- $distributorHost := include "loki.distributorFullname" .}} + {{- $ingesterHost := include "loki.ingesterFullname" .}} + {{- $queryFrontendHost := include "loki.queryFrontendFullname" .}} + {{- $indexGatewayHost := include "loki.indexGatewayFullname" .}} + {{- $rulerHost := include "loki.rulerFullname" .}} + {{- $compactorHost := include "loki.compactorFullname" .}} + {{- $schedulerHost := include "loki.querySchedulerFullname" .}} + + + {{- $distributorUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $distributorHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) -}} + {{- $ingesterUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $ingesterHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $queryFrontendUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $queryFrontendHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $indexGatewayUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $indexGatewayHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $rulerUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $rulerHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $compactorUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $compactorHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + {{- $schedulerUrl := printf "%s://%s.%s.svc.%s:%s" $httpSchema $schedulerHost .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.http_listen_port | toString) }} + + {{- if eq (include "loki.deployment.isSingleBinary" .) "true"}} + {{- $distributorUrl = $singleBinaryUrl }} + {{- $ingesterUrl = $singleBinaryUrl }} + {{- $queryFrontendUrl = $singleBinaryUrl }} + {{- $indexGatewayUrl = $singleBinaryUrl }} + {{- $rulerUrl = $singleBinaryUrl }} + {{- $compactorUrl = $singleBinaryUrl }} + {{- $schedulerUrl = $singleBinaryUrl }} + {{- else if eq (include "loki.deployment.isScalable" .) "true"}} + {{- $distributorUrl = $writeUrl }} + {{- $ingesterUrl = $writeUrl }} + {{- $queryFrontendUrl = $readUrl }} + {{- $indexGatewayUrl = $backendUrl }} + {{- $rulerUrl = $backendUrl }} + {{- $compactorUrl = $backendUrl }} + {{- $schedulerUrl = $backendUrl }} + {{- end -}} + + # Distributor + location = /api/prom/push { + proxy_pass {{ $distributorUrl }}$request_uri; + } + location = /loki/api/v1/push { + proxy_pass {{ $distributorUrl }}$request_uri; + } + location = /distributor/ring { + proxy_pass {{ $distributorUrl }}$request_uri; + } + location = /otlp/v1/logs { + proxy_pass {{ $distributorUrl }}$request_uri; + } + + # Ingester + location = /flush { + proxy_pass {{ $ingesterUrl }}$request_uri; + } + location ^~ /ingester/ { + proxy_pass {{ $ingesterUrl }}$request_uri; + } + location = /ingester { + internal; # to suppress 301 + } + + # Ring + location = /ring { + proxy_pass {{ $ingesterUrl }}$request_uri; + } + + # MemberListKV + location = /memberlist { + proxy_pass {{ $ingesterUrl }}$request_uri; + } + + # Ruler + location = /ruler/ring { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location = /api/prom/rules { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location ^~ /api/prom/rules/ { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location = /loki/api/v1/rules { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location ^~ /loki/api/v1/rules/ { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location = /prometheus/api/v1/alerts { + proxy_pass {{ $rulerUrl }}$request_uri; + } + location = /prometheus/api/v1/rules { + proxy_pass {{ $rulerUrl }}$request_uri; + } + + # Compactor + location = /compactor/ring { + proxy_pass {{ $compactorUrl }}$request_uri; + } + location = /loki/api/v1/delete { + proxy_pass {{ $compactorUrl }}$request_uri; + } + location = /loki/api/v1/cache/generation_numbers { + proxy_pass {{ $compactorUrl }}$request_uri; + } + + # IndexGateway + location = /indexgateway/ring { + proxy_pass {{ $indexGatewayUrl }}$request_uri; + } + + # QueryScheduler + location = /scheduler/ring { + proxy_pass {{ $schedulerUrl }}$request_uri; + } + + # Config + location = /config { + proxy_pass {{ $ingesterUrl }}$request_uri; + } + + {{- if and .Values.enterprise.enabled .Values.enterprise.adminApi.enabled }} + # Admin API + location ^~ /admin/api/ { + proxy_pass {{ $backendUrl }}$request_uri; + } + location = /admin/api { + internal; # to suppress 301 + } + {{- end }} + + + # QueryFrontend, Querier + location = /api/prom/tail { + proxy_pass {{ $queryFrontendUrl }}$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location = /loki/api/v1/tail { + proxy_pass {{ $queryFrontendUrl }}$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location ^~ /api/prom/ { + proxy_pass {{ $queryFrontendUrl }}$request_uri; + } + location = /api/prom { + internal; # to suppress 301 + } + location ^~ /loki/api/v1/ { + proxy_pass {{ $queryFrontendUrl }}$request_uri; + } + location = /loki/api/v1 { + internal; # to suppress 301 + } + + {{- with .Values.gateway.nginxConfig.serverSnippet }} + {{ . | nindent 4 }} + {{- end }} + } +} +{{- end }} + +{{/* Configure enableServiceLinks in pod */}} +{{- define "loki.enableServiceLinks" -}} +{{- if semverCompare ">=1.13-0" (include "loki.kubeVersion" .) -}} +{{- if or (.Values.loki.enableServiceLinks) (ne .Values.loki.enableServiceLinks false) -}} +enableServiceLinks: true +{{- else -}} +enableServiceLinks: false +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Determine compactor address based on target configuration */}} +{{- define "loki.compactorAddress" -}} +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- $compactorAddress := include "loki.backendFullname" . -}} +{{- if and $isSimpleScalable .Values.read.legacyReadTarget -}} +{{/* 2 target configuration */}} +{{- $compactorAddress = include "loki.readFullname" . -}} +{{- else if $isSingleBinary -}} +{{/* single binary */}} +{{- $compactorAddress = include "loki.singleBinaryFullname" . -}} +{{/* distributed */}} +{{- else if $isDistributed -}} +{{- $compactorAddress = include "loki.compactorFullname" . -}} +{{- end -}} +{{- printf "http://%s:%s" $compactorAddress (.Values.loki.server.http_listen_port | toString) }} +{{- end }} + +{{/* Determine query-scheduler address */}} +{{- define "loki.querySchedulerAddress" -}} +{{- $schedulerAddress := ""}} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +{{- $schedulerAddress = printf "%s.%s.svc.%s:%s" (include "loki.querySchedulerFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.grpc_listen_port | toString) -}} +{{- end -}} +{{- printf "%s" $schedulerAddress }} +{{- end }} + +{{/* Determine querier address */}} +{{- define "loki.querierAddress" -}} +{{- $querierAddress := "" }} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +{{- $querierHost := include "loki.querierFullname" .}} +{{- $querierUrl := printf "http://%s.%s.svc.%s:3100" $querierHost .Release.Namespace .Values.global.clusterDomain }} +{{- $querierAddress = $querierUrl }} +{{- end -}} +{{- printf "%s" $querierAddress }} +{{- end }} + +{{/* Determine index-gateway address */}} +{{- define "loki.indexGatewayAddress" -}} +{{- $idxGatewayAddress := ""}} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isDistributed -}} +{{- $idxGatewayAddress = printf "dns+%s-headless.%s.svc.%s:%s" (include "loki.indexGatewayFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.grpc_listen_port | toString) -}} +{{- end -}} +{{- if $isScalable -}} +{{- $idxGatewayAddress = printf "dns+%s-headless.%s.svc.%s:%s" (include "loki.backendFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.grpc_listen_port | toString) -}} +{{- end -}} +{{- printf "%s" $idxGatewayAddress }} +{{- end }} + +{{/* Determine bloom-planner address */}} +{{- define "loki.bloomPlannerAddress" -}} +{{- $bloomPlannerAddress := ""}} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isDistributed -}} +{{- $bloomPlannerAddress = printf "%s-headless.%s.svc.%s:%s" (include "loki.bloomPlannerFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.grpc_listen_port | toString) -}} +{{- end -}} +{{- if $isScalable -}} +{{- $bloomPlannerAddress = printf "%s-headless.%s.svc.%s:%s" (include "loki.backendFullname" .) .Release.Namespace .Values.global.clusterDomain (.Values.loki.server.grpc_listen_port | toString) -}} +{{- end -}} +{{- printf "%s" $bloomPlannerAddress}} +{{- end }} + +{{/* Determine bloom-gateway address */}} +{{- define "loki.bloomGatewayAddresses" -}} +{{- $bloomGatewayAddresses := ""}} +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isDistributed -}} +{{- $bloomGatewayAddresses = printf "dnssrvnoa+_grpc._tcp.%s-headless.%s.svc.%s" (include "loki.bloomGatewayFullname" .) .Release.Namespace .Values.global.clusterDomain -}} +{{- end -}} +{{- if $isScalable -}} +{{- $bloomGatewayAddresses = printf "dnssrvnoa+_grpc._tcp.%s-headless.%s.svc.%s" (include "loki.backendFullname" .) .Release.Namespace .Values.global.clusterDomain -}} +{{- end -}} +{{- printf "%s" $bloomGatewayAddresses}} +{{- end }} + +{{- define "loki.config.checksum" -}} +checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodDisruptionBudget. +*/}} +{{- define "loki.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">=1.21-0" (include "loki.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the object store type for use with the test schema. +*/}} +{{- define "loki.testSchemaObjectStore" -}} + {{- if .Values.minio.enabled -}} + s3 + {{- else -}} + filesystem + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for HorizontalPodAutoscaler. +*/}} +{{- define "loki.hpa.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "autoscaling/v2") (semverCompare ">= 1.19-0" (include "loki.kubeVersion" .)) -}} + {{- print "autoscaling/v2" -}} + {{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta2" -}} + {{- print "autoscaling/v2beta2" -}} + {{- else -}} + {{- print "autoscaling/v2beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +compute a ConfigMap or Secret checksum only based on its .data content. +This function needs to be called with a context object containing the following keys: +- ctx: the current Helm context (what '.' is at the call site) +- name: the file name of the ConfigMap or Secret +*/}} +{{- define "loki.configMapOrSecretContentHash" -}} +{{ get (include (print .ctx.Template.BasePath .name) .ctx | fromYaml) "data" | toYaml | sha256sum }} +{{- end }} diff --git a/opencloud/charts/loki/templates/admin-api/_helpers.yaml b/opencloud/charts/loki/templates/admin-api/_helpers.yaml new file mode 100644 index 0000000..e13ff8a --- /dev/null +++ b/opencloud/charts/loki/templates/admin-api/_helpers.yaml @@ -0,0 +1,24 @@ +{{/* +adminApi fullname +*/}} +{{- define "enterprise-logs.adminApiFullname" -}} +{{ include "loki.fullname" . }}-admin-api +{{- end }} + +{{/* +adminApi common labels +*/}} +{{- define "enterprise-logs.adminApiLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: admin-api +target: admin-api +{{- end }} + +{{/* +adminApi selector labels +*/}} +{{- define "enterprise-logs.adminApiSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: admin-api +target: admin-api +{{- end }} \ No newline at end of file diff --git a/opencloud/charts/loki/templates/admin-api/deployment-admin-api.yaml b/opencloud/charts/loki/templates/admin-api/deployment-admin-api.yaml new file mode 100644 index 0000000..c5e97dd --- /dev/null +++ b/opencloud/charts/loki/templates/admin-api/deployment-admin-api.yaml @@ -0,0 +1,176 @@ +{{- if .Values.enterprise.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "enterprise-logs.adminApiFullname" . }} + labels: + {{- include "enterprise-logs.adminApiLabels" . | nindent 4 }} + {{- with .Values.adminApi.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + annotations: + {{- with .Values.adminApi.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.adminApi.replicas }} + selector: + matchLabels: + {{- include "enterprise-logs.adminApiSelectorLabels" . | nindent 6 }} + strategy: + {{- toYaml .Values.adminApi.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "enterprise-logs.adminApiSelectorLabels" . | nindent 8 }} + {{- with .Values.adminApi.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- end}} + {{- with .Values.adminApi.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.adminApi.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ template "loki.serviceAccountName" . }} + {{- if .Values.adminApi.priorityClassName }} + priorityClassName: {{ .Values.adminApi.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.adminApi.podSecurityContext | nindent 8 }} + initContainers: + # Taken from + # https://github.com/minio/charts/blob/a5c84bcbad884728bff5c9c23541f936d57a13b3/minio/templates/post-install-create-bucket-job.yaml + {{- if .Values.minio.enabled }} + - name: minio-mc + image: "{{ .Values.minio.mcImage.repository }}:{{ .Values.minio.mcImage.tag }}" + imagePullPolicy: {{ .Values.minio.mcImage.pullPolicy }} + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: {{ .Release.Name }}-minio + - name: MINIO_PORT + value: {{ .Values.minio.service.port | quote }} + volumeMounts: + - name: minio-configuration + mountPath: /config + {{- if .Values.minio.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.minio.configPathmc }}certs + {{ end }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.adminApi.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: admin-api + image: "{{ template "loki.image" . }}" + imagePullPolicy: {{ .Values.enterprise.image.pullPolicy }} + args: + - -target=admin-api + - -config.file=/etc/loki/config/config.yaml + {{- if .Values.minio.enabled }} + - -admin.client.backend-type=s3 + - -admin.client.s3.endpoint={{ template "loki.minio" . }} + - -admin.client.s3.bucket-name=enterprise-logs-admin + - -admin.client.s3.access-key-id={{ .Values.minio.accessKey }} + - -admin.client.s3.secret-access-key={{ .Values.minio.secretKey }} + - -admin.client.s3.insecure=true + {{- end }} + {{- range $key, $value := .Values.adminApi.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: license + mountPath: /etc/loki/license + - name: storage + mountPath: /data + {{- if .Values.adminApi.extraVolumeMounts }} + {{ toYaml .Values.adminApi.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + readinessProbe: + {{- toYaml .Values.adminApi.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.adminApi.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.adminApi.containerSecurityContext | nindent 12 }} + env: + {{- if .Values.adminApi.env }} + {{ toYaml .Values.adminApi.env | nindent 12 }} + {{- end }} + {{- with .Values.adminApi.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.adminApi.extraContainers }} + {{ toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.adminApi.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.adminApi.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.adminApi.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.adminApi.terminationGracePeriodSeconds }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + - name: storage + emptyDir: {} + {{- if .Values.adminApi.extraVolumes }} + {{ toYaml .Values.adminApi.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.minio.enabled }} + - name: minio-configuration + projected: + sources: + - configMap: + name: {{ .Release.Name }}-minio + - secret: + name: {{ .Release.Name }}-minio + {{- if .Values.minio.tls.enabled }} + - name: cert-secret-volume-mc + secret: + secretName: {{ .Values.minio.tls.certSecret }} + items: + - key: {{ .Values.minio.tls.publicCrt }} + path: CAs/public.crt + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/admin-api/service-admin-api.yaml b/opencloud/charts/loki/templates/admin-api/service-admin-api.yaml new file mode 100644 index 0000000..c7daa27 --- /dev/null +++ b/opencloud/charts/loki/templates/admin-api/service-admin-api.yaml @@ -0,0 +1,28 @@ +{{- if .Values.enterprise.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "enterprise-logs.adminApiFullname" . }} + labels: + {{- include "enterprise-logs.adminApiLabels" . | nindent 4 }} + {{- with .Values.adminApi.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.adminApi.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + protocol: TCP + targetPort: http-metrics + - name: grpc + port: 9095 + protocol: TCP + targetPort: grpc + selector: + {{- include "enterprise-logs.adminApiSelectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/opencloud/charts/loki/templates/backend/_helpers-backend.tpl b/opencloud/charts/loki/templates/backend/_helpers-backend.tpl new file mode 100644 index 0000000..08f5f8f --- /dev/null +++ b/opencloud/charts/loki/templates/backend/_helpers-backend.tpl @@ -0,0 +1,32 @@ +{{/* +backend fullname +*/}} +{{- define "loki.backendFullname" -}} +{{ include "loki.name" . }}-backend +{{- end }} + +{{/* +backend common labels +*/}} +{{- define "loki.backendLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: backend +{{- end }} + +{{/* +backend selector labels +*/}} +{{- define "loki.backendSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: backend +{{- end }} + +{{/* +backend priority class name +*/}} +{{- define "loki.backendPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.backend.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/clusterrole.yaml b/opencloud/charts/loki/templates/backend/clusterrole.yaml new file mode 100644 index 0000000..36c8a0f --- /dev/null +++ b/opencloud/charts/loki/templates/backend/clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "loki.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "loki.fullname" . }}-clusterrole +{{- if .Values.sidecar.rules.enabled }} +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/clusterrolebinding.yaml b/opencloud/charts/loki/templates/backend/clusterrolebinding.yaml new file mode 100644 index 0000000..92f86a4 --- /dev/null +++ b/opencloud/charts/loki/templates/backend/clusterrolebinding.yaml @@ -0,0 +1,25 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if (not .Values.rbac.namespaced) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "loki.fullname" . }}-clusterrolebinding + labels: + {{- include "loki.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "loki.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole +{{- if (not .Values.rbac.useExistingRole) }} + name: {{ template "loki.fullname" . }}-clusterrole +{{- else }} + name: {{ .Values.rbac.useExistingRole }} +{{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/opencloud/charts/loki/templates/backend/hpa.yaml b/opencloud/charts/loki/templates/backend/hpa.yaml new file mode 100644 index 0000000..ea834d6 --- /dev/null +++ b/opencloud/charts/loki/templates/backend/hpa.yaml @@ -0,0 +1,50 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- $autoscalingv2 := .Capabilities.APIVersions.Has "autoscaling/v2" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) ( .Values.backend.autoscaling.enabled ) }} +{{- if $autoscalingv2 }} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.backendFullname" . }} + labels: + {{- include "loki.backendLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.backendFullname" . }} + minReplicas: {{ .Values.backend.autoscaling.minReplicas }} + maxReplicas: {{ .Values.backend.autoscaling.maxReplicas }} + {{- with .Values.backend.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.backend.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.backend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/poddisruptionbudget-backend.yaml b/opencloud/charts/loki/templates/backend/poddisruptionbudget-backend.yaml new file mode 100644 index 0000000..d8ce5b0 --- /dev/null +++ b/opencloud/charts/loki/templates/backend/poddisruptionbudget-backend.yaml @@ -0,0 +1,15 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (gt (int .Values.backend.replicas) 1) (not .Values.read.legacyReadTarget ) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.backendFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.backendLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/query-scheduler-discovery.yaml b/opencloud/charts/loki/templates/backend/query-scheduler-discovery.yaml new file mode 100644 index 0000000..4c357e5 --- /dev/null +++ b/opencloud/charts/loki/templates/backend/query-scheduler-discovery.yaml @@ -0,0 +1,34 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.querySchedulerFullname" . }}-discovery + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.backendSelectorLabels" . | nindent 4 }} + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.backend.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.backendSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/service-backend-headless.yaml b/opencloud/charts/loki/templates/backend/service-backend-headless.yaml new file mode 100644 index 0000000..0755be6 --- /dev/null +++ b/opencloud/charts/loki/templates/backend/service-backend-headless.yaml @@ -0,0 +1,40 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.backendFullname" . }}-headless + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.backendSelectorLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.backend.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + variant: headless + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.backend.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.backendSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/service-backend.yaml b/opencloud/charts/loki/templates/backend/service-backend.yaml new file mode 100644 index 0000000..cd1bd3b --- /dev/null +++ b/opencloud/charts/loki/templates/backend/service-backend.yaml @@ -0,0 +1,37 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.backendFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.backendLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.backend.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.backend.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.backendSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/backend/statefulset-backend.yaml b/opencloud/charts/loki/templates/backend/statefulset-backend.yaml new file mode 100644 index 0000000..bf2e18d --- /dev/null +++ b/opencloud/charts/loki/templates/backend/statefulset-backend.yaml @@ -0,0 +1,287 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.backendFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.backendLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.backend.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.backend.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: +{{- if not .Values.backend.autoscaling.enabled }} + {{- if eq .Values.deploymentMode "SingleBinary" }} + replicas: 0 + {{- else }} + replicas: {{ .Values.backend.replicas }} + {{- end }} +{{- end }} + podManagementPolicy: {{ .Values.backend.podManagementPolicy }} + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.backendFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.backend.persistence.enableStatefulSetAutoDeletePVC) (.Values.backend.persistence.volumeClaimsEnabled) }} + {{/* + Data on the backend nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + {{- end }} + selector: + matchLabels: + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.backendSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.backendPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.backend.terminationGracePeriodSeconds }} + {{- if .Values.backend.initContainers }} + initContainers: + {{- with .Values.backend.initContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + {{- if .Values.sidecar.rules.enabled }} + - name: loki-sc-rules + {{- if .Values.sidecar.image.sha }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.image.pullPolicy }} + env: + - name: METHOD + value: {{ .Values.sidecar.rules.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.rules.label }}" + {{- if .Values.sidecar.rules.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.rules.labelValue }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.rules.folder }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.rules.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.rules.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.rules.searchNamespace | join "," }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.rules.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.rules.script }}" + {{- end }} + {{- if .Values.sidecar.rules.watchServerTimeout }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.rules.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.watchClientTimeout }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.rules.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.logLevel }} + - name: LOG_LEVEL + value: "{{ .Values.sidecar.rules.logLevel }}" + {{- end }} + {{- if .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml .Values.sidecar.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml .Values.sidecar.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.resources }} + resources: + {{- toYaml .Values.sidecar.resources | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.securityContext }} + securityContext: + {{- toYaml .Values.sidecar.securityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} + - name: loki + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.backend.targetModule }} + - -legacy-read-mode=false + {{- with .Values.backend.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.backend.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: tmp + mountPath: /tmp + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end}} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} + {{- with .Values.backend.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.backend.resources | nindent 12 }} + {{- with .Values.backend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.backend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if not .Values.backend.persistence.volumeClaimsEnabled }} + - name: data + {{- toYaml .Values.backend.persistence.dataVolumeParameters | nindent 10 }} + {{- end}} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + {{- if .Values.sidecar.rules.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.rules.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end -}} + {{- end -}} + {{- with .Values.backend.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.backend.persistence.volumeClaimsEnabled }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + {{- with .Values.backend.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.backend.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.backend.persistence.size | quote }} + {{- with .Values.backend.persistence.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/bloom-builder/_helpers-bloom-builder.tpl b/opencloud/charts/loki/templates/bloom-builder/_helpers-bloom-builder.tpl new file mode 100644 index 0000000..46359df --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/_helpers-bloom-builder.tpl @@ -0,0 +1,32 @@ +{{/* +bloom-builder fullname +*/}} +{{- define "loki.bloomBuilderFullname" -}} +{{ include "loki.fullname" . }}-bloom-builder +{{- end }} + +{{/* +bloom-builder common labels +*/}} +{{- define "loki.bloomBuilderLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: bloom-builder +{{- end }} + +{{/* +bloom-builder selector labels +*/}} +{{- define "loki.bloomBuilderSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: bloom-builder +{{- end }} + +{{/* +bloom-builder priority class name +*/}} +{{- define "loki.bloomBuilderPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.bloomBuilder.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/bloom-builder/deployment-bloom-builder.yaml b/opencloud/charts/loki/templates/bloom-builder/deployment-bloom-builder.yaml new file mode 100644 index 0000000..c04b3ae --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/deployment-bloom-builder.yaml @@ -0,0 +1,150 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (gt (int .Values.bloomPlanner.replicas) 0)) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.bloomBuilderFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomBuilderLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.bloomBuilder.autoscaling.enabled }} + replicas: {{ .Values.bloomBuilder.replicas }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.bloomBuilderSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomBuilder.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.bloomBuilderSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomBuilder.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomBuilder.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.bloomBuilderPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.bloomBuilder.terminationGracePeriodSeconds }} + containers: + - name: bloom-builder + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.bloomBuilder.command }} + command: + - {{ coalesce .Values.bloomBuilder.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=bloom-builder + {{- with .Values.bloomBuilder.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.bloomBuilder.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.bloomBuilder.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + - name: temp + mountPath: /tmp + - name: data + mountPath: /var/loki + {{- with .Values.bloomBuilder.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.bloomBuilder.resources | nindent 12 }} + {{- if .Values.bloomBuilder.extraContainers }} + {{- toYaml .Values.bloomBuilder.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.bloomBuilder.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomBuilder.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomBuilder.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + - name: temp + emptyDir: {} + - name: data + emptyDir: {} + {{- with .Values.bloomBuilder.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-builder/hpa.yaml b/opencloud/charts/loki/templates/bloom-builder/hpa.yaml new file mode 100644 index 0000000..2b04647 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/hpa.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.bloomBuilder.autoscaling.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.bloomBuilderFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomBuilderLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "loki.bloomBuilderFullname" . }} + minReplicas: {{ .Values.bloomBuilder.autoscaling.minReplicas }} + maxReplicas: {{ .Values.bloomBuilder.autoscaling.maxReplicas }} + metrics: + {{- with .Values.bloomBuilder.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.bloomBuilder.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.bloomBuilder.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.bloomBuilder.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.bloomBuilder.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.bloomBuilder.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/bloom-builder/poddisruptionbudget-bloom-builder.yaml b/opencloud/charts/loki/templates/bloom-builder/poddisruptionbudget-bloom-builder.yaml new file mode 100644 index 0000000..e66d762 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/poddisruptionbudget-bloom-builder.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.bloomBuilder.replicas) 1) }} +{{- if kindIs "invalid" .Values.bloomBuilder.maxUnavailable }} +{{- fail "`.Values.bloomBuilder.maxUnavailable` must be set when `.Values.bloomBuilder.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.bloomBuilderFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomBuilderLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.bloomBuilderSelectorLabels" . | nindent 6 }} + {{- with .Values.bloomBuilder.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder-headless.yaml b/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder-headless.yaml new file mode 100644 index 0000000..9389252 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder-headless.yaml @@ -0,0 +1,46 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (or (gt (int .Values.bloomBuilder.replicas) 0)) .Values.bloomBuilder.autoscaling.enabled) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.bloomBuilderFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomBuilderLabels" . | nindent 4 }} + {{- with .Values.bloomBuilder.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.bloomBuilder.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + type: ClusterIP + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomBuilder.appProtocol.grpc }} + appProtocol: {{ .Values.bloomBuilder.appProtocol.grpc }} + {{- end }} + - name: grpclb + port: 9096 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomBuilder.appProtocol.grpc }} + appProtocol: {{ .Values.bloomBuilder.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.bloomBuilderSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder.yaml b/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder.yaml new file mode 100644 index 0000000..b3debb0 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-builder/service-bloom-builder.yaml @@ -0,0 +1,44 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (gt (int .Values.bloomBuilder.replicas) 0)) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.bloomBuilderFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomBuilderLabels" . | nindent 4 }} + {{- with .Values.bloomBuilder.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.bloomBuilder.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomBuilder.appProtocol.grpc }} + appProtocol: {{ .Values.bloomBuilder.appProtocol.grpc }} + {{- end }} + - name: grpclb + port: 9096 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomBuilder.appProtocol.grpc }} + appProtocol: {{ .Values.bloomBuilder.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.bloomBuilderSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-gateway/_helpers-bloom-gateway.tpl b/opencloud/charts/loki/templates/bloom-gateway/_helpers-bloom-gateway.tpl new file mode 100644 index 0000000..f0cef4f --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-gateway/_helpers-bloom-gateway.tpl @@ -0,0 +1,58 @@ +{{/* +bloom gateway fullname +*/}} +{{- define "loki.bloomGatewayFullname" -}} +{{ include "loki.fullname" . }}-bloom-gateway +{{- end }} + +{{/* +bloom gateway common labels +*/}} +{{- define "loki.bloomGatewayLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: bloom-gateway +{{- end }} + +{{/* +bloom gateway selector labels +*/}} +{{- define "loki.bloomGatewaySelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: bloom-gateway +{{- end }} + +{{/* +bloom gateway readinessProbe +*/}} +{{- define "loki.bloomGateway.readinessProbe" -}} +{{- with .Values.bloomGateway.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +bloom gateway priority class name +*/}} +{{- define "loki.bloomGatewayPriorityClassName" }} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.bloomGateway.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* +Create the name of the bloom gateway service account +*/}} +{{- define "loki.bloomGatewayServiceAccountName" -}} +{{- if .Values.bloomGateway.serviceAccount.create -}} + {{ default (print (include "loki.serviceAccountName" .) "-bloom-gateway") .Values.bloomGateway.serviceAccount.name }} +{{- else -}} + {{ default (include "loki.serviceAccountName" .) .Values.bloomGateway.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-gateway/service-bloom-gateway-headless.yaml b/opencloud/charts/loki/templates/bloom-gateway/service-bloom-gateway-headless.yaml new file mode 100644 index 0000000..852e4cb --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-gateway/service-bloom-gateway-headless.yaml @@ -0,0 +1,39 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +{{- if (gt (int .Values.bloomGateway.replicas) 0) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.bloomGatewayFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomGatewaySelectorLabels" . | nindent 4 }} + {{- with .Values.bloomGateway.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.bloomGateway.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomGateway.appProtocol.grpc }} + appProtocol: {{ .Values.bloomGateway.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.bloomGatewaySelectorLabels" . | nindent 4 }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-gateway/statefulset-bloom-gateway.yaml b/opencloud/charts/loki/templates/bloom-gateway/statefulset-bloom-gateway.yaml new file mode 100644 index 0000000..747642b --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-gateway/statefulset-bloom-gateway.yaml @@ -0,0 +1,181 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (gt (int .Values.bloomGateway.replicas) 0)) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.bloomGatewayFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomGatewayLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.bloomGateway.replicas }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.bloomGatewayFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.bloomGateway.persistence.enableStatefulSetAutoDeletePVC) }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.bloomGateway.persistence.whenDeleted }} + whenScaled: {{ .Values.bloomGateway.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.bloomGatewaySelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomGateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.bloomGatewaySelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomGateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomGateway.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.bloomGatewayPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.bloomGateway.terminationGracePeriodSeconds }} + {{- with .Values.bloomGateway.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: bloom-gateway + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.bloomGateway.command }} + command: + - {{ coalesce .Values.bloomGateway.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=bloom-gateway + {{- with .Values.bloomGateway.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.bloomGateway.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.bloomGateway.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.bloomGateway.readinessProbe" . | nindent 10 }} + volumeMounts: + - name: temp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.bloomGateway.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.bloomGateway.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.bloomGateway.extraContainers }} + {{- toYaml .Values.bloomGateway.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.bloomGateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomGateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomGateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: temp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if not .Values.bloomGateway.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.bloomGateway.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.bloomGateway.persistence.enabled }} + volumeClaimTemplates: + {{- range .Values.bloomGateway.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-planner/_helpers-bloom-planner.tpl b/opencloud/charts/loki/templates/bloom-planner/_helpers-bloom-planner.tpl new file mode 100644 index 0000000..a4a8c6e --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-planner/_helpers-bloom-planner.tpl @@ -0,0 +1,58 @@ +{{/* +bloom planner fullname +*/}} +{{- define "loki.bloomPlannerFullname" -}} +{{ include "loki.fullname" . }}-bloom-planner +{{- end }} + +{{/* +bloom planner common labels +*/}} +{{- define "loki.bloomPlannerLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: bloom-planner +{{- end }} + +{{/* +bloom planner selector labels +*/}} +{{- define "loki.bloomPlannerSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: bloom-planner +{{- end }} + +{{/* +bloom planner readinessProbe +*/}} +{{- define "loki.bloomPlanner.readinessProbe" -}} +{{- with .Values.bloomPlanner.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +bloom planner priority class name +*/}} +{{- define "loki.bloomPlannerPriorityClassName" }} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.bloomPlanner.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* +Create the name of the bloom planner service account +*/}} +{{- define "loki.bloomPlannerServiceAccountName" -}} +{{- if .Values.bloomPlanner.serviceAccount.create -}} + {{ default (print (include "loki.serviceAccountName" .) "-bloom-planner") .Values.bloomPlanner.serviceAccount.name }} +{{- else -}} + {{ default (include "loki.serviceAccountName" .) .Values.bloomPlanner.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-planner/service-bloom-planner-headless.yaml b/opencloud/charts/loki/templates/bloom-planner/service-bloom-planner-headless.yaml new file mode 100644 index 0000000..78e2633 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-planner/service-bloom-planner-headless.yaml @@ -0,0 +1,37 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (gt (int .Values.bloomPlanner.replicas) 0)) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.bloomPlannerFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomPlannerSelectorLabels" . | nindent 4 }} + {{- with .Values.bloomPlanner.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.bloomPlanner.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.bloomPlanner.appProtocol.grpc }} + appProtocol: {{ .Values.bloomPlanner.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.bloomPlannerSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/bloom-planner/statefulset-bloom-planner.yaml b/opencloud/charts/loki/templates/bloom-planner/statefulset-bloom-planner.yaml new file mode 100644 index 0000000..d134af3 --- /dev/null +++ b/opencloud/charts/loki/templates/bloom-planner/statefulset-bloom-planner.yaml @@ -0,0 +1,181 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if (and $isDistributed (gt (int .Values.bloomPlanner.replicas) 0)) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.bloomPlannerFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.bloomPlannerLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.bloomPlanner.replicas }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.bloomPlannerFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.bloomPlanner.persistence.enableStatefulSetAutoDeletePVC) }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.bloomPlanner.persistence.whenDeleted }} + whenScaled: {{ .Values.bloomPlanner.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.bloomPlannerSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomPlanner.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.bloomPlannerSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomPlanner.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomPlanner.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.bloomPlannerPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.bloomPlanner.terminationGracePeriodSeconds }} + {{- with .Values.bloomPlanner.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: bloom-planner + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.bloomPlanner.command }} + command: + - {{ coalesce .Values.bloomPlanner.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=bloom-planner + {{- with .Values.bloomPlanner.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.bloomPlanner.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.bloomPlanner.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.bloomPlanner.readinessProbe" . | nindent 10 }} + volumeMounts: + - name: temp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.bloomPlanner.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.bloomPlanner.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.bloomPlanner.extraContainers }} + {{- toYaml .Values.bloomPlanner.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.bloomPlanner.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomPlanner.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.bloomPlanner.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: temp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if not .Values.bloomPlanner.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.bloomPlanner.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.bloomPlanner.persistence.enabled }} + volumeClaimTemplates: + {{- range .Values.bloomPlanner.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/chunks-cache/poddisruptionbudget-chunks-cache.yaml b/opencloud/charts/loki/templates/chunks-cache/poddisruptionbudget-chunks-cache.yaml new file mode 100644 index 0000000..da95adf --- /dev/null +++ b/opencloud/charts/loki/templates/chunks-cache/poddisruptionbudget-chunks-cache.yaml @@ -0,0 +1,16 @@ +{{- if .Values.chunksCache.enabled }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.fullname" . }}-memcached-chunks-cache + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: memcached-chunks-cache +spec: + selector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: memcached-chunks-cache + maxUnavailable: 1 +{{- end -}} diff --git a/opencloud/charts/loki/templates/chunks-cache/service-chunks-cache-headless.yaml b/opencloud/charts/loki/templates/chunks-cache/service-chunks-cache-headless.yaml new file mode 100644 index 0000000..dc2ccd4 --- /dev/null +++ b/opencloud/charts/loki/templates/chunks-cache/service-chunks-cache-headless.yaml @@ -0,0 +1 @@ +{{- include "loki.memcached.service" (dict "ctx" $ "valuesSection" "chunksCache" "component" "chunks-cache" ) }} diff --git a/opencloud/charts/loki/templates/chunks-cache/statefulset-chunks-cache.yaml b/opencloud/charts/loki/templates/chunks-cache/statefulset-chunks-cache.yaml new file mode 100644 index 0000000..6a54c57 --- /dev/null +++ b/opencloud/charts/loki/templates/chunks-cache/statefulset-chunks-cache.yaml @@ -0,0 +1 @@ +{{- include "loki.memcached.statefulSet" (dict "ctx" $ "valuesSection" "chunksCache" "component" "chunks-cache" ) }} diff --git a/opencloud/charts/loki/templates/ciliumnetworkpolicy.yaml b/opencloud/charts/loki/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000..fbd2619 --- /dev/null +++ b/opencloud/charts/loki/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,238 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "cilium") }} +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-namespace-only + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: {} + egress: + - toEndpoints: + - {} + ingress: + - fromEndpoints: + - {} + +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-dns + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - toPorts: + - ports: + - port: dns + protocol: UDP + toEndpoints: + - namespaceSelector: {} + +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-ingress + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + {{- if .Values.gateway.enabled }} + - gateway + {{- else }} + - read + - write + {{- end }} + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + ingress: + - toPorts: + - ports: + - port: http + protocol: TCP + {{- if .Values.networkPolicy.ingress.namespaceSelector }} + fromEndpoints: + - matchLabels: + {{- toYaml .Values.networkPolicy.ingress.namespaceSelector | nindent 8 }} + {{- if .Values.networkPolicy.ingress.podSelector }} + {{- toYaml .Values.networkPolicy.ingress.podSelector | nindent 8 }} + {{- end }} + {{- end }} + +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-ingress-metrics + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + ingress: + - toPorts: + - ports: + - port: http-metrics + protocol: TCP + {{- if .Values.networkPolicy.metrics.cidrs }} + {{- range $cidr := .Values.networkPolicy.metrics.cidrs }} + toCIDR: + - {{ $cidr }} + {{- end }} + {{- if .Values.networkPolicy.metrics.namespaceSelector }} + fromEndpoints: + - matchLabels: + {{- toYaml .Values.networkPolicy.metrics.namespaceSelector | nindent 8 }} + {{- if .Values.networkPolicy.metrics.podSelector }} + {{- toYaml .Values.networkPolicy.metrics.podSelector | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} + +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-alertmanager + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + egress: + - toPorts: + - ports: + - port: "{{ .Values.networkPolicy.alertmanager.port }}" + protocol: TCP + {{- if .Values.networkPolicy.alertmanager.namespaceSelector }} + toEndpoints: + - matchLabels: + {{- toYaml .Values.networkPolicy.alertmanager.namespaceSelector | nindent 8 }} + {{- if .Values.networkPolicy.alertmanager.podSelector }} + {{- toYaml .Values.networkPolicy.alertmanager.podSelector | nindent 8 }} + {{- end }} + {{- end }} + +{{- if .Values.networkPolicy.externalStorage.ports }} +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-external-storage + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - toPorts: + - ports: + {{- range $port := .Values.networkPolicy.externalStorage.ports }} + - port: "{{ $port }}" + protocol: TCP + {{- end }} + {{- if .Values.networkPolicy.externalStorage.cidrs }} + {{- range $cidr := .Values.networkPolicy.externalStorage.cidrs }} + toCIDR: + - {{ $cidr }} + {{- end }} + {{- end }} +{{- end }} + +{{- if .Values.networkPolicy.egressWorld.enabled }} +{{- $global := . }} +{{- $componentsList := list "read" "write" "backend" }} +{{- if .Values.tableManager.enabled }} +{{- $componentsList = append $componentsList "table-manager" }} +{{- end }} +{{- range $component := $componentsList }} +{{- with $global }} +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-{{ $component }}-world-egress + namespace: {{ .Release.Namespace }} +spec: + endpointSelector: + matchLabels: + {{- if eq $component "read" }} + {{- include "loki.readSelectorLabels" . | nindent 6 }} + {{- else if eq $component "write" }} + {{- include "loki.writeSelectorLabels" . | nindent 6 }} + {{- else if eq $component "table-manager" }} + {{- include "loki.tableManagerSelectorLabels" . | nindent 6 }} + {{- else }} + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + {{- end }} + egress: + - toEntities: + - world +{{- end }} +{{- end }} +{{- end }} + +{{- if .Values.networkPolicy.egressKubeApiserver.enabled }} +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-backend-kubeapiserver-egress + namespace: {{ .Release.Namespace }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + egress: + - toEntities: + - kube-apiserver +{{- end }} + +{{- end }} + +{{- if and .Values.networkPolicy.discovery.port (eq .Values.networkPolicy.flavor "cilium") }} +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-discovery + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - toPorts: + - ports: + - port: "{{ .Values.networkPolicy.discovery.port }}" + protocol: TCP + {{- if .Values.networkPolicy.discovery.namespaceSelector }} + toEndpoints: + - matchLabels: + {{- toYaml .Values.networkPolicy.discovery.namespaceSelector | nindent 8 }} + {{- if .Values.networkPolicy.discovery.podSelector }} + {{- toYaml .Values.networkPolicy.discovery.podSelector | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/compactor/_helpers-compactor.tpl b/opencloud/charts/loki/templates/compactor/_helpers-compactor.tpl new file mode 100644 index 0000000..75c21db --- /dev/null +++ b/opencloud/charts/loki/templates/compactor/_helpers-compactor.tpl @@ -0,0 +1,81 @@ +{{/* +compactor fullname +*/}} +{{- define "loki.compactorFullname" -}} +{{ include "loki.fullname" . }}-compactor +{{- end }} + +{{/* +compactor common labels +*/}} +{{- define "loki.compactorLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: compactor +{{- end }} + +{{/* +compactor selector labels +*/}} +{{- define "loki.compactorSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: compactor +{{- end }} + +{{/* +compactor image +*/}} +{{- define "loki.compactorImage" -}} +{{- $dict := dict "loki" .Values.loki.image "service" .Values.compactor.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.lokiImage" $dict -}} +{{- end }} + +{{/* +compactor readinessProbe +*/}} +{{- define "loki.compactor.readinessProbe" -}} +{{- with .Values.compactor.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +compactor livenessProbe +*/}} +{{- define "loki.compactor.livenessProbe" -}} +{{- with .Values.compactor.livenessProbe }} +livenessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.livenessProbe }} +livenessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +compactor priority class name +*/}} +{{- define "loki.compactorPriorityClassName" }} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.compactor.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* +Create the name of the compactor service account +*/}} +{{- define "loki.compactorServiceAccountName" -}} +{{- if .Values.compactor.serviceAccount.create -}} + {{ default (print (include "loki.serviceAccountName" .) "-compactor") .Values.compactor.serviceAccount.name }} +{{- else -}} + {{ default (include "loki.serviceAccountName" .) .Values.compactor.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/compactor/service-compactor.yaml b/opencloud/charts/loki/templates/compactor/service-compactor.yaml new file mode 100644 index 0000000..f118b6c --- /dev/null +++ b/opencloud/charts/loki/templates/compactor/service-compactor.yaml @@ -0,0 +1,38 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.compactorFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.compactor.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + app.kubernetes.io/component: compactor + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.compactor.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.compactor.appProtocol.grpc }} + appProtocol: {{ .Values.compactor.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: compactor +{{- end }} diff --git a/opencloud/charts/loki/templates/compactor/statefulset-compactor.yaml b/opencloud/charts/loki/templates/compactor/statefulset-compactor.yaml new file mode 100644 index 0000000..ded01be --- /dev/null +++ b/opencloud/charts/loki/templates/compactor/statefulset-compactor.yaml @@ -0,0 +1,193 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.compactorFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.compactorLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.compactor.replicas }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.compactorFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.compactor.persistence.enableStatefulSetAutoDeletePVC) }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.compactor.persistence.whenDeleted }} + whenScaled: {{ .Values.compactor.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.compactorSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.compactor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.compactorSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.compactor.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.compactor.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.compactor.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.compactorPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.compactor.terminationGracePeriodSeconds }} + {{- with .Values.compactor.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: compactor + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.compactor.command }} + command: + - {{ coalesce .Values.compactor.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=compactor + {{- with .Values.compactor.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.compactor.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.compactor.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.compactor.readinessProbe" . | nindent 10 }} + {{- include "loki.compactor.livenessProbe" . | nindent 10 }} + volumeMounts: + - name: temp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.compactor.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.compactor.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.compactor.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.compactor.extraContainers }} + {{- toYaml .Values.compactor.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.compactor.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.compactor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.compactor.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: temp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if not .Values.compactor.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.compactor.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.compactor.persistence.enabled }} + volumeClaimTemplates: + {{- range .Values.compactor.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/config.yaml b/opencloud/charts/loki/templates/config.yaml new file mode 100644 index 0000000..fe47590 --- /dev/null +++ b/opencloud/charts/loki/templates/config.yaml @@ -0,0 +1,21 @@ +{{- if .Values.loki.generatedConfigObjectName -}} +apiVersion: v1 +{{- if eq .Values.loki.configStorageType "Secret" }} +kind: Secret +{{- else }} +kind: ConfigMap +{{- end }} +metadata: + name: {{ tpl .Values.loki.generatedConfigObjectName . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +{{- if eq .Values.loki.configStorageType "Secret" }} +data: + config.yaml: {{ include "loki.calculatedConfig" . | b64enc }} +{{- else }} +data: + config.yaml: | + {{ include "loki.calculatedConfig" . | nindent 4 }} +{{- end -}} +{{- end }} diff --git a/opencloud/charts/loki/templates/distributor/_helpers-distributor.tpl b/opencloud/charts/loki/templates/distributor/_helpers-distributor.tpl new file mode 100644 index 0000000..c23179e --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/_helpers-distributor.tpl @@ -0,0 +1,32 @@ +{{/* +distributor fullname +*/}} +{{- define "loki.distributorFullname" -}} +{{ include "loki.fullname" . }}-distributor +{{- end }} + +{{/* +distributor common labels +*/}} +{{- define "loki.distributorLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: distributor +{{- end }} + +{{/* +distributor selector labels +*/}} +{{- define "loki.distributorSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: distributor +{{- end }} + +{{/* +distributor priority class name +*/}} +{{- define "loki.distributorPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.distributor.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/distributor/deployment-distributor.yaml b/opencloud/charts/loki/templates/distributor/deployment-distributor.yaml new file mode 100644 index 0000000..556a053 --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/deployment-distributor.yaml @@ -0,0 +1,158 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.distributorFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.distributorLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.distributor.autoscaling.enabled }} + replicas: {{ .Values.distributor.replicas }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.distributor.maxSurge }} + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.distributorSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.distributor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.distributorSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.distributor.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.distributor.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.distributor.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.distributorPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.distributor.terminationGracePeriodSeconds }} + containers: + - name: distributor + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.distributor.command }} + command: + - {{ coalesce .Values.distributor.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=distributor + {{- if .Values.ingester.zoneAwareReplication.enabled }} + {{- if and (.Values.ingester.zoneAwareReplication.migration.enabled) (not .Values.ingester.zoneAwareReplication.migration.writePath) }} + - -distributor.zone-awareness-enabled=false + {{- else }} + - -distributor.zone-awareness-enabled=true + {{- end }} + {{- end }} + {{- with .Values.distributor.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.distributor.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.distributor.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.distributor.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.distributor.resources | nindent 12 }} + {{- if .Values.distributor.extraContainers }} + {{- toYaml .Values.distributor.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.distributor.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.distributor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.distributor.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.distributor.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/distributor/hpa.yaml b/opencloud/charts/loki/templates/distributor/hpa.yaml new file mode 100644 index 0000000..838a310 --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/hpa.yaml @@ -0,0 +1,54 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.distributor.autoscaling.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.distributorFullname" . }} + labels: + {{- include "loki.distributorLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "loki.distributorFullname" . }} + minReplicas: {{ .Values.distributor.autoscaling.minReplicas }} + maxReplicas: {{ .Values.distributor.autoscaling.maxReplicas }} + metrics: + {{- with .Values.distributor.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.distributor.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.distributor.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.distributor.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.distributor.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.distributor.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/distributor/poddisruptionbudget-distributor.yaml b/opencloud/charts/loki/templates/distributor/poddisruptionbudget-distributor.yaml new file mode 100644 index 0000000..806a447 --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/poddisruptionbudget-distributor.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.distributor.replicas) 1) }} +{{- if kindIs "invalid" .Values.distributor.maxUnavailable }} +{{- fail "`.Values.distributor.maxUnavailable` must be set when `.Values.distributor.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.distributorFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.distributorLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.distributorSelectorLabels" . | nindent 6 }} + {{- with .Values.distributor.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/distributor/service-distributor-headless.yaml b/opencloud/charts/loki/templates/distributor/service-distributor-headless.yaml new file mode 100644 index 0000000..650b629 --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/service-distributor-headless.yaml @@ -0,0 +1,39 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.distributorFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.distributorSelectorLabels" . | nindent 4 }} + {{- with .Values.distributor.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + variant: headless + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.distributor.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.distributor.appProtocol.grpc }} + appProtocol: {{ .Values.distributor.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.distributorSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/distributor/service-distributor.yaml b/opencloud/charts/loki/templates/distributor/service-distributor.yaml new file mode 100644 index 0000000..6a89956 --- /dev/null +++ b/opencloud/charts/loki/templates/distributor/service-distributor.yaml @@ -0,0 +1,36 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.distributorFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.distributorLabels" . | nindent 4 }} + {{- with .Values.distributor.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.distributor.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.distributor.appProtocol.grpc }} + appProtocol: {{ .Values.distributor.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.distributorSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/extra-manifests.yaml b/opencloud/charts/loki/templates/extra-manifests.yaml new file mode 100644 index 0000000..7b69423 --- /dev/null +++ b/opencloud/charts/loki/templates/extra-manifests.yaml @@ -0,0 +1,8 @@ +{{- range .Values.extraObjects }} +--- +{{- if kindIs "map" . }} +{{ tpl (toYaml .) $ }} +{{- else }} +{{ tpl . $ }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/_helpers-gateway.tpl b/opencloud/charts/loki/templates/gateway/_helpers-gateway.tpl new file mode 100644 index 0000000..39890b1 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/_helpers-gateway.tpl @@ -0,0 +1,47 @@ +{{/* +gateway fullname +*/}} +{{- define "loki.gatewayFullname" -}} +{{ include "loki.fullname" . }}-gateway +{{- end }} + +{{/* +gateway common labels +*/}} +{{- define "loki.gatewayLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: gateway +{{- end }} + +{{/* +gateway selector labels +*/}} +{{- define "loki.gatewaySelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: gateway +{{- end }} + +{{/* +gateway auth secret name +*/}} +{{- define "loki.gatewayAuthSecret" -}} +{{ .Values.gateway.basicAuth.existingSecret | default (include "loki.gatewayFullname" . ) }} +{{- end }} + +{{/* +gateway Docker image +*/}} +{{- define "loki.gatewayImage" -}} +{{- $dict := dict "service" .Values.gateway.image "global" .Values.global.image -}} +{{- include "loki.baseImage" $dict -}} +{{- end }} + +{{/* +gateway priority class name +*/}} +{{- define "loki.gatewayPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.gateway.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/configmap-gateway.yaml b/opencloud/charts/loki/templates/gateway/configmap-gateway.yaml new file mode 100644 index 0000000..1c981a7 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/configmap-gateway.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.gateway.enabled (not (and .Values.enterprise.enabled .Values.enterprise.gelGateway)) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} +data: + nginx.conf: | + {{- tpl .Values.gateway.nginxConfig.file . | indent 2 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/deployment-gateway-enterprise.yaml b/opencloud/charts/loki/templates/gateway/deployment-gateway-enterprise.yaml new file mode 100644 index 0000000..22bf5c1 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/deployment-gateway-enterprise.yaml @@ -0,0 +1,152 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and .Values.gateway.enabled .Values.enterprise.enabled .Values.enterprise.gelGateway }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "loki.gatewayFullname" . }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} + {{- with .Values.enterpriseGateway.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterpriseGateway.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.enterpriseGateway.replicas }} + selector: + matchLabels: + {{- include "loki.gatewaySelectorLabels" . | nindent 6 }} + strategy: + {{- toYaml .Values.enterpriseGateway.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "loki.gatewaySelectorLabels" . | nindent 8 }} + {{- with .Values.enterpriseGateway.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- end}} + {{- with .Values.enterpriseGateway.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.enterpriseGateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ template "loki.serviceAccountName" . }} + {{- if .Values.enterpriseGateway.priorityClassName }} + priorityClassName: {{ .Values.enterpriseGateway.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.enterpriseGateway.podSecurityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.enterpriseGateway.initContainers | nindent 8 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterpriseGateway.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: gateway + image: "{{ template "loki.image" . }}" + imagePullPolicy: {{ .Values.enterprise.image.pullPolicy }} + args: + - -target=gateway + - -config.file=/etc/loki/config/config.yaml + {{- if .Values.minio.enabled }} + - -admin.client.backend-type=s3 + - -admin.client.s3.endpoint={{ template "loki.minio" . }} + - -admin.client.s3.bucket-name=enterprise-logs-admin + - -admin.client.s3.access-key-id={{ .Values.minio.accessKey }} + - -admin.client.s3.secret-access-key={{ .Values.minio.secretKey }} + - -admin.client.s3.insecure=true + {{- end }} + {{- if and $isDistributed .Values.enterpriseGateway.useDefaultProxyURLs }} + - -gateway.proxy.default.url=http://{{ template "loki.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.admin-api.url=http://{{ template "loki.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.distributor.url=dns:///{{ template "loki.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc:9095 + - -gateway.proxy.ingester.url=http://{{ template "loki.fullname" . }}-ingester.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.query-frontend.url=http://{{ template "loki.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.ruler.url=http://{{ template "loki.fullname" . }}-ruler.{{ .Release.Namespace }}.svc:3100 + {{- end }} + {{- if and $isSimpleScalable .Values.enterpriseGateway.useDefaultProxyURLs }} + - -gateway.proxy.default.url=http://{{ template "enterprise-logs.adminApiFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.admin-api.url=http://{{ template "enterprise-logs.adminApiFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.compactor.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.distributor.url=dns:///{{ template "loki.writeFullname" . }}-headless.{{ .Release.Namespace }}.svc:9095 + - -gateway.proxy.ingester.url=http://{{ template "loki.writeFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.query-frontend.url=http://{{ template "loki.readFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.ruler.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.query-scheduler.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + {{- end }} + {{- range $key, $value := .Values.enterpriseGateway.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: license + mountPath: /etc/loki/license + - name: storage + mountPath: /data + {{- if .Values.enterpriseGateway.extraVolumeMounts }} + {{ toYaml .Values.enterpriseGateway.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + readinessProbe: + {{- toYaml .Values.enterpriseGateway.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.enterpriseGateway.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.enterpriseGateway.containerSecurityContext | nindent 12 }} + env: + {{- if .Values.enterpriseGateway.env }} + {{ toYaml .Values.enterpriseGateway.env | nindent 12 }} + {{- end }} + {{- with .Values.enterpriseGateway.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.enterpriseGateway.extraContainers }} + {{ toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.enterpriseGateway.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.enterpriseGateway.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.enterpriseGateway.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.enterpriseGateway.terminationGracePeriodSeconds }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + - name: storage + emptyDir: {} + {{- if .Values.enterpriseGateway.extraVolumes }} + {{ toYaml .Values.enterpriseGateway.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/deployment-gateway-nginx.yaml b/opencloud/charts/loki/templates/gateway/deployment-gateway-nginx.yaml new file mode 100644 index 0000000..94562ad --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/deployment-gateway-nginx.yaml @@ -0,0 +1,138 @@ +{{- if and .Values.gateway.enabled (not (and .Values.enterprise.enabled .Values.enterprise.gelGateway)) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.backend.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gateway.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: +{{- if not .Values.gateway.autoscaling.enabled }} + replicas: {{ .Values.gateway.replicas }} +{{- end }} +{{- with .Values.gateway.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.gatewaySelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/gateway/configmap-gateway.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.gatewaySelectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{ include "loki.enableServiceLinks" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end -}} + {{- include "loki.gatewayPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.gateway.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.gateway.terminationGracePeriodSeconds }} + containers: + - name: nginx + image: {{ include "loki.gatewayImage" . }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + ports: + - name: http-metrics + containerPort: {{ .Values.gateway.containerPort }} + protocol: TCP + {{- with .Values.gateway.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + {{- toYaml .Values.gateway.readinessProbe | nindent 12 }} + securityContext: + {{- toYaml .Values.gateway.containerSecurityContext | nindent 12 }} + {{- with .Values.gateway.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/nginx + {{- if .Values.gateway.basicAuth.enabled }} + - name: auth + mountPath: /etc/nginx/secrets + {{- end }} + - name: tmp + mountPath: /tmp + - name: docker-entrypoint-d-override + mountPath: /docker-entrypoint.d + {{- if .Values.gateway.extraVolumeMounts }} + {{- toYaml .Values.gateway.extraVolumeMounts | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.gateway.resources | nindent 12 }} + {{- if .Values.gateway.extraContainers }} + {{- toYaml .Values.gateway.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "loki.gatewayFullname" . }} + {{- if .Values.gateway.basicAuth.enabled }} + - name: auth + secret: + secretName: {{ include "loki.gatewayAuthSecret" . }} + {{- end }} + - name: tmp + emptyDir: {} + - name: docker-entrypoint-d-override + emptyDir: {} + {{- if .Values.gateway.extraVolumes }} + {{- toYaml .Values.gateway.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/hpa.yaml b/opencloud/charts/loki/templates/gateway/hpa.yaml new file mode 100644 index 0000000..3541ec6 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/hpa.yaml @@ -0,0 +1,50 @@ +{{- $autoscalingv2 := .Capabilities.APIVersions.Has "autoscaling/v2" -}} +{{- if .Values.gateway.autoscaling.enabled }} +{{- if $autoscalingv2 }} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "loki.gatewayFullname" . }} + minReplicas: {{ .Values.gateway.autoscaling.minReplicas }} + maxReplicas: {{ .Values.gateway.autoscaling.maxReplicas }} + {{- with .Values.gateway.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.gateway.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.gateway.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/ingress-gateway.yaml b/opencloud/charts/loki/templates/gateway/ingress-gateway.yaml new file mode 100644 index 0000000..6f18e33 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/ingress-gateway.yaml @@ -0,0 +1,59 @@ +{{- if and .Values.gateway.enabled -}} +{{- if .Values.gateway.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "loki.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "loki.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "loki.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "loki.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} + {{- range $labelKey, $labelValue := .Values.gateway.ingress.labels }} + {{ $labelKey }}: {{ $labelValue | toYaml }} + {{- end }} + {{- with .Values.gateway.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.gateway.ingress.ingressClassName }} + ingressClassName: {{ .Values.gateway.ingress.ingressClassName }} + {{- end -}} + {{- if .Values.gateway.ingress.tls }} + tls: + {{- range .Values.gateway.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ tpl . $ | quote }} + {{- end }} + {{- with .secretName }} + secretName: {{ . }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.gateway.ingress.hosts }} + - host: {{ tpl .host $ | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if $ingressSupportsPathType }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ include "loki.gatewayFullname" $ }} + port: + number: {{ $.Values.gateway.service.port }} + {{- else }} + serviceName: {{ include "loki.gatewayFullname" $ }} + servicePort: {{ $.Values.gateway.service.port }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/poddisruptionbudget-gateway.yaml b/opencloud/charts/loki/templates/gateway/poddisruptionbudget-gateway.yaml new file mode 100644 index 0000000..0057c56 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/poddisruptionbudget-gateway.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.gateway.enabled }} +{{- if or + (and (not .Values.gateway.autoscaling.enabled) (gt (int .Values.gateway.replicas) 1)) + (and .Values.gateway.autoscaling.enabled (gt (int .Values.gateway.autoscaling.minReplicas) 1)) +}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.gatewaySelectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/secret-gateway.yaml b/opencloud/charts/loki/templates/gateway/secret-gateway.yaml new file mode 100644 index 0000000..c3c5e9a --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/secret-gateway.yaml @@ -0,0 +1,14 @@ +{{- with .Values.gateway }} +{{- if and .enabled .basicAuth.enabled (not .basicAuth.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "loki.gatewayFullname" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" $ | nindent 4 }} +stringData: + .htpasswd: | + {{- tpl .basicAuth.htpasswd $ | nindent 4 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/gateway/service-gateway.yaml b/opencloud/charts/loki/templates/gateway/service-gateway.yaml new file mode 100644 index 0000000..8c71026 --- /dev/null +++ b/opencloud/charts/loki/templates/gateway/service-gateway.yaml @@ -0,0 +1,40 @@ +{{- if .Values.gateway.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.gatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.gatewayLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.gateway.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.gateway.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: {{ .Values.gateway.service.type }} + {{- with .Values.gateway.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- if and (eq "LoadBalancer" .Values.gateway.service.type) .Values.gateway.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.gateway.service.loadBalancerIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.gateway.service.port }} + targetPort: http-metrics + {{- if and (eq "NodePort" .Values.gateway.service.type) .Values.gateway.service.nodePort }} + nodePort: {{ .Values.gateway.service.nodePort }} + {{- end }} + protocol: TCP + selector: + {{- include "loki.gatewaySelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/index-gateway/_helpers-index-gateway.tpl b/opencloud/charts/loki/templates/index-gateway/_helpers-index-gateway.tpl new file mode 100644 index 0000000..f42dff3 --- /dev/null +++ b/opencloud/charts/loki/templates/index-gateway/_helpers-index-gateway.tpl @@ -0,0 +1,40 @@ +{{/* +index-gateway fullname +*/}} +{{- define "loki.indexGatewayFullname" -}} +{{ include "loki.fullname" . }}-index-gateway +{{- end }} + +{{/* +index-gateway common labels +*/}} +{{- define "loki.indexGatewayLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: index-gateway +{{- end }} + +{{/* +index-gateway selector labels +*/}} +{{- define "loki.indexGatewaySelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: index-gateway +{{- end }} + +{{/* +index-gateway image +*/}} +{{- define "loki.indexGatewayImage" -}} +{{- $dict := dict "loki" .Values.loki.image "service" .Values.indexGateway.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.lokiImage" $dict -}} +{{- end }} + +{{/* +index-gateway priority class name +*/}} +{{- define "loki.indexGatewayPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.indexGateway.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/index-gateway/poddisruptionbudget-index-gateway.yaml b/opencloud/charts/loki/templates/index-gateway/poddisruptionbudget-index-gateway.yaml new file mode 100644 index 0000000..cc230c2 --- /dev/null +++ b/opencloud/charts/loki/templates/index-gateway/poddisruptionbudget-index-gateway.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.indexGateway.replicas) 1) }} +{{- if kindIs "invalid" .Values.indexGateway.maxUnavailable }} +{{- fail "`.Values.indexGateway.maxUnavailable` must be set when `.Values.indexGateway.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.indexGatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.indexGatewayLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 6 }} + {{- with .Values.indexGateway.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/index-gateway/service-index-gateway-headless.yaml b/opencloud/charts/loki/templates/index-gateway/service-index-gateway-headless.yaml new file mode 100644 index 0000000..731a984 --- /dev/null +++ b/opencloud/charts/loki/templates/index-gateway/service-index-gateway-headless.yaml @@ -0,0 +1,35 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.indexGatewayFullname" . }}-headless + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 4 }} + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.indexGateway.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- with .Values.indexGateway.appProtocol.grpc }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/index-gateway/service-index-gateway.yaml b/opencloud/charts/loki/templates/index-gateway/service-index-gateway.yaml new file mode 100644 index 0000000..b1a3523 --- /dev/null +++ b/opencloud/charts/loki/templates/index-gateway/service-index-gateway.yaml @@ -0,0 +1,36 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.indexGatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.indexGatewayLabels" . | nindent 4 }} + {{- with .Values.indexGateway.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.indexGateway.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- with .Values.indexGateway.appProtocol.grpc }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/index-gateway/statefulset-index-gateway.yaml b/opencloud/charts/loki/templates/index-gateway/statefulset-index-gateway.yaml new file mode 100644 index 0000000..e249859 --- /dev/null +++ b/opencloud/charts/loki/templates/index-gateway/statefulset-index-gateway.yaml @@ -0,0 +1,192 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.indexGatewayFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.indexGatewayLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.indexGateway.replicas }} +{{- with .Values.indexGateway.updateStrategy }} + updateStrategy: + {{- tpl (. | toYaml) $ | nindent 4 }} +{{- end }} + serviceName: {{ include "loki.indexGatewayFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.indexGateway.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.indexGateway.persistence.whenDeleted }} + whenScaled: {{ .Values.indexGateway.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.indexGatewaySelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.indexGateway.joinMemberlist }} + app.kubernetes.io/part-of: memberlist + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.indexGatewayPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.indexGateway.terminationGracePeriodSeconds }} + {{- with .Values.indexGateway.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: index-gateway + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=index-gateway + {{- with .Values.indexGateway.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + {{- if .Values.indexGateway.joinMemberlist }} + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- end }} + {{- with .Values.indexGateway.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.indexGateway.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.indexGateway.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.indexGateway.resources | nindent 12 }} + {{- if .Values.indexGateway.extraContainers }} + {{- toYaml .Values.indexGateway.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.indexGateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexGateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.indexGateway.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.indexGateway.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if .Values.indexGateway.persistence.inMemory }} + - name: data + {{- if .Values.indexGateway.persistence.inMemory }} + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.indexGateway.persistence.size }} + sizeLimit: {{ .Values.indexGateway.persistence.size }} + {{- end }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: data + {{- with .Values.indexGateway.persistence.annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.indexGateway.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.indexGateway.persistence.size | quote }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/_helpers-ingester.tpl b/opencloud/charts/loki/templates/ingester/_helpers-ingester.tpl new file mode 100644 index 0000000..418d409 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/_helpers-ingester.tpl @@ -0,0 +1,74 @@ +{{/* +ingester fullname +*/}} +{{- define "loki.ingesterFullname" -}} +{{ include "loki.fullname" . }}-ingester +{{- end }} + +{{/* +ingester common labels +*/}} +{{- define "loki.ingesterLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: ingester +{{- end }} + +{{/* +ingester selector labels +*/}} +{{- define "loki.ingesterSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: ingester +{{- end }} + +{{/* +ingester priority class name +*/}} +{{- define "loki.ingesterPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.ingester.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{- define "loki.ingester.readinessProbe" -}} +{{- with .Values.ingester.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "loki.ingester.livenessProbe" -}} +{{- with .Values.ingester.livenessProbe }} +livenessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.livenessProbe }} +livenessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +expects global context +*/}} +{{- define "loki.ingester.replicaCount" -}} +{{- ceil (divf .Values.ingester.replicas 3) -}} +{{- end -}} + +{{/* +expects a dict +{ + "replicas": replicas in a zone, + "ctx": global context +} +*/}} +{{- define "loki.ingester.maxUnavailable" -}} +{{- ceil (mulf .replicas (divf (int .ctx.Values.ingester.zoneAwareReplication.maxUnavailablePct) 100)) -}} +{{- end -}} \ No newline at end of file diff --git a/opencloud/charts/loki/templates/ingester/hpa-zone-a.yaml b/opencloud/charts/loki/templates/ingester/hpa-zone-a.yaml new file mode 100644 index 0000000..5ede187 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/hpa-zone-a.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.autoscaling.enabled .Values.ingester.zoneAwareReplication.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-a + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.ingesterFullname" . }}-zone-a + minReplicas: {{ .Values.ingester.autoscaling.minReplicas }} + maxReplicas: {{ .Values.ingester.autoscaling.maxReplicas }} + metrics: + {{- with .Values.ingester.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.ingester.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.ingester.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.ingester.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/hpa-zone-b.yaml b/opencloud/charts/loki/templates/ingester/hpa-zone-b.yaml new file mode 100644 index 0000000..b001a6a --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/hpa-zone-b.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.autoscaling.enabled .Values.ingester.zoneAwareReplication.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-b + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.ingesterFullname" . }}-zone-b + minReplicas: {{ .Values.ingester.autoscaling.minReplicas }} + maxReplicas: {{ .Values.ingester.autoscaling.maxReplicas }} + metrics: + {{- with .Values.ingester.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.ingester.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.ingester.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.ingester.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/hpa-zone-c.yaml b/opencloud/charts/loki/templates/ingester/hpa-zone-c.yaml new file mode 100644 index 0000000..82f229c --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/hpa-zone-c.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.autoscaling.enabled .Values.ingester.zoneAwareReplication.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-c + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.ingesterFullname" . }}-zone-c + minReplicas: {{ .Values.ingester.autoscaling.minReplicas }} + maxReplicas: {{ .Values.ingester.autoscaling.maxReplicas }} + metrics: + {{- with .Values.ingester.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.ingester.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.ingester.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.ingester.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/hpa.yaml b/opencloud/charts/loki/templates/ingester/hpa.yaml new file mode 100644 index 0000000..de35d67 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/hpa.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.autoscaling.enabled (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.ingesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.ingesterFullname" . }} + minReplicas: {{ .Values.ingester.autoscaling.minReplicas }} + maxReplicas: {{ .Values.ingester.autoscaling.maxReplicas }} + metrics: + {{- with .Values.ingester.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.ingester.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.ingester.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.ingester.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml b/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml new file mode 100644 index 0000000..000ab85 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.ingester.replicas) 1) (.Values.ingester.zoneAwareReplication.enabled) }} +{{- if kindIs "invalid" .Values.ingester.maxUnavailable }} +{{- fail "`.Values.ingester.maxUnavailable` must be set when `.Values.ingester.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.ingesterFullname" . }}-rollout + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + rollout-group: ingester + {{- with .Values.ingester.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester.yaml b/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester.yaml new file mode 100644 index 0000000..1142c01 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/poddisruptionbudget-ingester.yaml @@ -0,0 +1,27 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.ingester.replicas) 1) (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }} +{{- if kindIs "invalid" .Values.ingester.maxUnavailable }} +{{- fail "`.Values.ingester.maxUnavailable` must be set when `.Values.ingester.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.ingesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.ingesterSelectorLabels" . | nindent 6 }} + {{/* zone aware ingesters get their own pod disruption budget, ignore them here */}} + matchExpressions: + - key: rollout-group + operator: NotIn + values: + - "ingester" + {{- with .Values.ingester.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/service-ingester-headless.yaml b/opencloud/charts/loki/templates/ingester/service-ingester-headless.yaml new file mode 100644 index 0000000..8a8b92f --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/service-ingester-headless.yaml @@ -0,0 +1,35 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.ingesterFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ingester.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.ingester.appProtocol.grpc }} + appProtocol: {{ .Values.ingester.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/ingester/service-ingester-zone-a-headless.yaml b/opencloud/charts/loki/templates/ingester/service-ingester-zone-a-headless.yaml new file mode 100644 index 0000000..03add3b --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/service-ingester-zone-a-headless.yaml @@ -0,0 +1,38 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-a-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + {{- with .Values.ingester.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ingester.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.ingester.appProtocol.grpc }} + appProtocol: {{ .Values.ingester.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} + name: ingester-zone-a + rollout-group: ingester +{{- end -}} diff --git a/opencloud/charts/loki/templates/ingester/service-ingester-zone-b-headless.yaml b/opencloud/charts/loki/templates/ingester/service-ingester-zone-b-headless.yaml new file mode 100644 index 0000000..6072219 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/service-ingester-zone-b-headless.yaml @@ -0,0 +1,38 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-b-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + {{- with .Values.ingester.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ingester.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.ingester.appProtocol.grpc }} + appProtocol: {{ .Values.ingester.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} + name: ingester-zone-b + rollout-group: ingester +{{- end -}} diff --git a/opencloud/charts/loki/templates/ingester/service-ingester-zone-c-headless.yaml b/opencloud/charts/loki/templates/ingester/service-ingester-zone-c-headless.yaml new file mode 100644 index 0000000..5541447 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/service-ingester-zone-c-headless.yaml @@ -0,0 +1,38 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-c-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + {{- with .Values.ingester.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ingester.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.ingester.appProtocol.grpc }} + appProtocol: {{ .Values.ingester.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} + name: ingester-zone-c + rollout-group: ingester +{{- end -}} diff --git a/opencloud/charts/loki/templates/ingester/service-ingester.yaml b/opencloud/charts/loki/templates/ingester/service-ingester.yaml new file mode 100644 index 0000000..94d6f83 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/service-ingester.yaml @@ -0,0 +1,36 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.ingesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + {{- with .Values.ingester.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ingester.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.ingester.appProtocol.grpc }} + appProtocol: {{ .Values.ingester.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.ingesterSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-a.yaml b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-a.yaml new file mode 100644 index 0000000..a731661 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-a.yaml @@ -0,0 +1,234 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +{{- $replicas := (include "loki.ingester.replicaCount" .) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-a + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + rollout-group: ingester + name: ingester-zone-a + annotations: + rollout-max-unavailable: "{{ include "loki.ingester.maxUnavailable" (dict "ctx" . "replicas" $replicas)}}" + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneA.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.ingester.autoscaling.enabled }} + replicas: {{ $replicas }} +{{- end }} + podManagementPolicy: Parallel + serviceName: {{ include "loki.ingesterFullname" . }}-zone-a + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.ingester.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.ingester.persistence.whenDeleted }} + whenScaled: {{ .Values.ingester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.ingesterSelectorLabels" . | nindent 6 }} + name: ingester-zone-a + rollout-group: ingester +{{- with .Values.ingester.updateStrategy }} + updateStrategy: + {{- tpl (. | toYaml) $ | nindent 4 }} +{{- end }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneA.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.ingesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + name: ingester-zone-a + rollout-group: ingester + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.ingesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }} + {{- with .Values.ingester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.ingester.command }} + command: + - {{ coalesce .Values.ingester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -ingester.availability-zone=zone-a + - -ingester.unregister-on-shutdown=false + - -ingester.tokens-file-path=/var/loki/ring-tokens + - -target=ingester + {{- with .Values.ingester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.ingester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.ingester.readinessProbe" . | nindent 10 }} + {{- include "loki.ingester.livenessProbe" . | nindent 10 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.ingester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.ingester.extraContainers }} + {{- toYaml .Values.ingester.extraContainers | nindent 8}} + {{- end }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-a + topologyKey: kubernetes.io/hostname + {{- with .Values.ingester.zoneAwareReplication.zoneA.extraAffinity }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneA.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.ingester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.ingester.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if .Values.ingester.persistence.inMemory }} + - name: data + {{- if .Values.ingester.persistence.inMemory }} + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.ingester.persistence.size }} + sizeLimit: {{ .Values.ingester.persistence.size }} + {{- end }} + {{- else }} + volumeClaimTemplates: + {{- range .Values.ingester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-b.yaml b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-b.yaml new file mode 100644 index 0000000..7512454 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-b.yaml @@ -0,0 +1,234 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +{{- $replicas := (include "loki.ingester.replicaCount" .) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-b + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + rollout-group: ingester + name: ingester-zone-b + annotations: + rollout-max-unavailable: "{{ include "loki.ingester.maxUnavailable" (dict "ctx" . "replicas" $replicas)}}" + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneB.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.ingester.autoscaling.enabled }} + replicas: {{ $replicas }} +{{- end }} + podManagementPolicy: Parallel + serviceName: {{ include "loki.ingesterFullname" . }}-zone-b + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.ingester.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.ingester.persistence.whenDeleted }} + whenScaled: {{ .Values.ingester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.ingesterSelectorLabels" . | nindent 6 }} + name: ingester-zone-b + rollout-group: ingester +{{- with .Values.ingester.updateStrategy }} + updateStrategy: + {{- tpl (. | toYaml) $ | nindent 4 }} +{{- end }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneB.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.ingesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + name: ingester-zone-b + rollout-group: ingester + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.ingesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }} + {{- with .Values.ingester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.ingester.command }} + command: + - {{ coalesce .Values.ingester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -ingester.availability-zone=zone-b + - -ingester.unregister-on-shutdown=false + - -ingester.tokens-file-path=/var/loki/ring-tokens + - -target=ingester + {{- with .Values.ingester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.ingester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.ingester.readinessProbe" . | nindent 10 }} + {{- include "loki.ingester.livenessProbe" . | nindent 10 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.ingester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.ingester.extraContainers }} + {{- toYaml .Values.ingester.extraContainers | nindent 8}} + {{- end }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-b + topologyKey: kubernetes.io/hostname + {{- with .Values.ingester.zoneAwareReplication.zoneB.extraAffinity }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneB.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.ingester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.ingester.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if .Values.ingester.persistence.inMemory }} + - name: data + {{- if .Values.ingester.persistence.inMemory }} + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.ingester.persistence.size }} + sizeLimit: {{ .Values.ingester.persistence.size }} + {{- end }} + {{- else }} + volumeClaimTemplates: + {{- range .Values.ingester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-c.yaml b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-c.yaml new file mode 100644 index 0000000..657d6a5 --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/statefulset-ingester-zone-c.yaml @@ -0,0 +1,234 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }} +{{- $replicas := (include "loki.ingester.replicaCount" .) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.ingesterFullname" . }}-zone-c + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + rollout-group: ingester + name: ingester-zone-c + annotations: + rollout-max-unavailable: "{{ include "loki.ingester.maxUnavailable" (dict "ctx" . "replicas" $replicas)}}" + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneC.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.ingester.autoscaling.enabled }} + replicas: {{ $replicas }} +{{- end }} + podManagementPolicy: Parallel + serviceName: {{ include "loki.ingesterFullname" . }}-zone-c + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.ingester.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.ingester.persistence.whenDeleted }} + whenScaled: {{ .Values.ingester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.ingesterSelectorLabels" . | nindent 6 }} + name: ingester-zone-c + rollout-group: ingester +{{- with .Values.ingester.updateStrategy }} + updateStrategy: + {{- tpl (. | toYaml) $ | nindent 4 }} +{{- end }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneC.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.ingesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + name: ingester-zone-c + rollout-group: ingester + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.ingesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }} + {{- with .Values.ingester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.ingester.command }} + command: + - {{ coalesce .Values.ingester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -ingester.availability-zone=zone-c + - -ingester.unregister-on-shutdown=false + - -ingester.tokens-file-path=/var/loki/ring-tokens + - -target=ingester + {{- with .Values.ingester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.ingester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.ingester.readinessProbe" . | nindent 10 }} + {{- include "loki.ingester.livenessProbe" . | nindent 10 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.ingester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.ingester.extraContainers }} + {{- toYaml .Values.ingester.extraContainers | nindent 8}} + {{- end }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-c + topologyKey: kubernetes.io/hostname + {{- with .Values.ingester.zoneAwareReplication.zoneC.extraAffinity }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.zoneAwareReplication.zoneC.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.ingester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.ingester.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if .Values.ingester.persistence.inMemory }} + - name: data + {{- if .Values.ingester.persistence.inMemory }} + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.ingester.persistence.size }} + sizeLimit: {{ .Values.ingester.persistence.size }} + {{- end }} + {{- else }} + volumeClaimTemplates: + {{- range .Values.ingester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingester/statefulset-ingester.yaml b/opencloud/charts/loki/templates/ingester/statefulset-ingester.yaml new file mode 100644 index 0000000..217196a --- /dev/null +++ b/opencloud/charts/loki/templates/ingester/statefulset-ingester.yaml @@ -0,0 +1,205 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.ingesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.ingesterLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.ingester.autoscaling.enabled }} + replicas: {{ .Values.ingester.replicas }} +{{- end }} + podManagementPolicy: Parallel +{{- with .Values.ingester.updateStrategy }} + updateStrategy: + {{- tpl (. | toYaml) $ | nindent 4 }} +{{- end }} + serviceName: {{ include "loki.ingesterFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.ingester.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.ingester.persistence.whenDeleted }} + whenScaled: {{ .Values.ingester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.ingesterSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.ingesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.ingesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }} + {{- with .Values.ingester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.ingester.command }} + command: + - {{ coalesce .Values.ingester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -ingester.availability-zone=zone-default + - -target=ingester + {{- with .Values.ingester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.ingester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.ingester.readinessProbe" . | nindent 10 }} + {{- include "loki.ingester.livenessProbe" . | nindent 10 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.ingester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.ingester.extraContainers }} + {{- toYaml .Values.ingester.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.ingester.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ingester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.ingester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.ingester.persistence.enabled }} + - name: data + emptyDir: { } + {{- else if .Values.ingester.persistence.inMemory }} + - name: data + {{- if .Values.ingester.persistence.inMemory }} + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.ingester.persistence.size }} + sizeLimit: {{ .Values.ingester.persistence.size }} + {{- end }} + {{- else }} + volumeClaimTemplates: + {{- range .Values.ingester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ingress.yaml b/opencloud/charts/loki/templates/ingress.yaml new file mode 100644 index 0000000..ddbcf7f --- /dev/null +++ b/opencloud/charts/loki/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled }} +{{- $ingressSupportsIngressClassName := eq (include "loki.ingress.supportsIngressClassName" .) "true" -}} +apiVersion: {{ include "loki.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "loki.fullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ tpl . $ | quote }} + {{- end }} + {{- with .secretName }} + secretName: {{ . }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range $.Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- include "loki.ingress.servicePaths" $ | indent 10}} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/loki-canary/_helpers.tpl b/opencloud/charts/loki/templates/loki-canary/_helpers.tpl new file mode 100644 index 0000000..01e588c --- /dev/null +++ b/opencloud/charts/loki/templates/loki-canary/_helpers.tpl @@ -0,0 +1,40 @@ +{{/* +canary fullname +*/}} +{{- define "loki-canary.fullname" -}} +{{ include "loki.name" . }}-canary +{{- end }} + +{{/* +canary common labels +*/}} +{{- define "loki-canary.labels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: canary +{{- end }} + +{{/* +canary selector labels +*/}} +{{- define "loki-canary.selectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: canary +{{- end }} + +{{/* +Docker image name for loki-canary +*/}} +{{- define "loki-canary.image" -}} +{{- $dict := dict "service" .Values.lokiCanary.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + +{{/* +canary priority class name +*/}} +{{- define "loki-canary.priorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.lokiCanary.priorityClassName .Values.read.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/loki-canary/daemonset.yaml b/opencloud/charts/loki/templates/loki-canary/daemonset.yaml new file mode 100644 index 0000000..dc5c629 --- /dev/null +++ b/opencloud/charts/loki/templates/loki-canary/daemonset.yaml @@ -0,0 +1,123 @@ +{{- with .Values.lokiCanary -}} +{{- if .enabled -}} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "loki-canary.fullname" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki-canary.labels" $ | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki-canary.selectorLabels" $ | nindent 6 }} + {{- with .updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki-canary.selectorLabels" $ | nindent 8 }} + {{- with .podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki-canary.fullname" $ }} + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki-canary.priorityClassName" $ | nindent 6 }} + securityContext: + {{- toYaml $.Values.loki.podSecurityContext | nindent 8 }} + containers: + - name: loki-canary + image: {{ include "loki-canary.image" $ }} + imagePullPolicy: {{ $.Values.loki.image.pullPolicy }} + args: + - -addr={{- include "loki.host" $ }} + - -labelname={{ .labelname }} + - -labelvalue=$(POD_NAME) + {{- if $.Values.enterprise.enabled }} + - -user=$(USER) + - -tenant-id=$(USER) + - -pass=$(PASS) + {{- else if $.Values.loki.auth_enabled }} + - -user={{ $.Values.monitoring.selfMonitoring.tenant.name }} + - -tenant-id={{ $.Values.monitoring.selfMonitoring.tenant.name }} + - -pass={{ $.Values.monitoring.selfMonitoring.tenant.password }} + {{- end }} + {{- if .push }} + - -push=true + {{- end }} + {{- with .extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml $.Values.loki.containerSecurityContext | nindent 12 }} + volumeMounts: + {{- with $.Values.lokiCanary.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3500 + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{ if $.Values.enterprise.enabled }} + - name: USER + valueFrom: + secretKeyRef: + name: {{ include "enterprise-logs.selfMonitoringTenantSecret" $ }} + key: username + - name: PASS + valueFrom: + secretKeyRef: + name: {{ include "enterprise-logs.selfMonitoringTenantSecret" $ }} + key: password + {{- end -}} + {{- with .extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + httpGet: + path: /metrics + port: http-metrics + initialDelaySeconds: 15 + timeoutSeconds: 1 + {{- with .resources}} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- with $.Values.lokiCanary.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/loki-canary/service.yaml b/opencloud/charts/loki/templates/loki-canary/service.yaml new file mode 100644 index 0000000..38022a3 --- /dev/null +++ b/opencloud/charts/loki/templates/loki-canary/service.yaml @@ -0,0 +1,34 @@ +{{- with .Values.lokiCanary -}} +{{- if .enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki-canary.fullname" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki-canary.labels" $ | nindent 4 }} + {{- with $.Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with $.Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3500 + targetPort: http-metrics + protocol: TCP + selector: + {{- include "loki-canary.selectorLabels" $ | nindent 4 }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/loki-canary/serviceaccount.yaml b/opencloud/charts/loki/templates/loki-canary/serviceaccount.yaml new file mode 100644 index 0000000..2c1f79a --- /dev/null +++ b/opencloud/charts/loki/templates/loki-canary/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- with .Values.lokiCanary -}} +{{- if .enabled -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "loki-canary.fullname" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki-canary.labels" $ | nindent 4 }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ $.Values.serviceAccount.automountServiceAccountToken }} +{{- with $.Values.serviceAccount.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/memcached/_memcached-statefulset.tpl b/opencloud/charts/loki/templates/memcached/_memcached-statefulset.tpl new file mode 100644 index 0000000..0664ba4 --- /dev/null +++ b/opencloud/charts/loki/templates/memcached/_memcached-statefulset.tpl @@ -0,0 +1,178 @@ +{{/* +memcached StatefulSet +Params: + ctx = . context + valuesSection = name of the section in values.yaml + component = name of the component +valuesSection and component are specified separately because helm prefers camelcase for naming convetion and k8s components are named with snake case. +*/}} +{{- define "loki.memcached.statefulSet" -}} +{{ with (index $.ctx.Values $.valuesSection) }} +{{- if .enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.resourceName" (dict "ctx" $.ctx "component" $.component) }} + labels: + {{- include "loki.labels" $.ctx | nindent 4 }} + app.kubernetes.io/component: "memcached-{{ $.component }}" + name: "memcached-{{ $.component }}" + annotations: + {{- toYaml .annotations | nindent 4 }} + namespace: {{ $.ctx.Release.Namespace | quote }} +spec: + podManagementPolicy: {{ .podManagementPolicy }} + replicas: {{ .replicas }} + selector: + matchLabels: + {{- include "loki.selectorLabels" $.ctx | nindent 6 }} + app.kubernetes.io/component: "memcached-{{ $.component }}" + name: "memcached-{{ $.component }}" + updateStrategy: + {{- toYaml .statefulStrategy | nindent 4 }} + serviceName: {{ template "loki.fullname" $.ctx }}-{{ $.component }} + + template: + metadata: + labels: + {{- include "loki.selectorLabels" $.ctx | nindent 8 }} + app.kubernetes.io/component: "memcached-{{ $.component }}" + name: "memcached-{{ $.component }}" + {{- with $.ctx.Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with $.ctx.Values.global.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + + spec: + serviceAccountName: {{ template "loki.serviceAccountName" $.ctx }} + {{- if .priorityClassName }} + priorityClassName: {{ .priorityClassName }} + {{- end }} + securityContext: + {{- toYaml $.ctx.Values.memcached.podSecurityContext | nindent 8 }} + initContainers: + {{- toYaml .initContainers | nindent 8 }} + nodeSelector: + {{- toYaml .nodeSelector | nindent 8 }} + affinity: + {{- toYaml .affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .topologySpreadConstraints | nindent 8 }} + tolerations: + {{- toYaml .tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds }} + {{- with $.ctx.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .extraVolumes }} + volumes: + {{- toYaml .extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if .extraContainers }} + {{ toYaml .extraContainers | nindent 8 }} + {{- end }} + - name: memcached + {{- with $.ctx.Values.memcached.image }} + image: {{ .repository }}:{{ .tag }} + imagePullPolicy: {{ .pullPolicy }} + {{- end }} + resources: + {{- if .resources }} + {{- toYaml .resources | nindent 12 }} + {{- else }} + {{- /* Calculate requested memory as round(allocatedMemory * 1.2). But with integer built-in operators. */}} + {{- $requestMemory := div (add (mul .allocatedMemory 12) 5) 10 }} + limits: + memory: {{ $requestMemory }}Mi + requests: + cpu: 500m + memory: {{ $requestMemory }}Mi + {{- end }} + ports: + - containerPort: {{ .port }} + name: client + {{- /* Calculate storage size as round(.persistence.storageSize * 0.9). But with integer built-in operators. */}} + {{- $persistenceSize := (div (mul (trimSuffix "Gi" .persistence.storageSize | trimSuffix "G") 9) 10 ) }} + args: + - -m {{ .allocatedMemory }} + - --extended=modern,track_sizes{{ if .persistence.enabled }},ext_path={{ .persistence.mountPath }}/file:{{ $persistenceSize }}G,ext_wbuf_size=16{{ end }}{{ with .extraExtendedOptions }},{{ . }}{{ end }} + - -I {{ .maxItemMemory }}m + - -c {{ .connectionLimit }} + - -v + - -u {{ .port }} + {{- range $key, $value := .extraArgs }} + - "-{{ $key }}{{ if $value }} {{ $value }}{{ end }}" + {{- end }} + env: + {{- with $.ctx.Values.global.extraEnv }} + {{ toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with $.ctx.Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml $.ctx.Values.memcached.containerSecurityContext | nindent 12 }} + {{- if or .persistence.enabled .extraVolumeMounts }} + volumeMounts: + {{- if .persistence.enabled }} + - name: data + mountPath: {{ .persistence.mountPath }} + {{- end }} + {{- if .extraVolumeMounts }} + {{- toYaml .extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + + {{- if $.ctx.Values.memcachedExporter.enabled }} + - name: exporter + {{- with $.ctx.Values.memcachedExporter.image }} + image: {{ .repository}}:{{ .tag }} + imagePullPolicy: {{ .pullPolicy }} + {{- end }} + ports: + - containerPort: 9150 + name: http-metrics + args: + - "--memcached.address=localhost:{{ .port }}" + - "--web.listen-address=0.0.0.0:9150" + {{- range $key, $value := $.ctx.Values.memcachedExporter.extraArgs }} + - "--{{ $key }}{{ if $value }}={{ $value }}{{ end }}" + {{- end }} + resources: + {{- toYaml $.ctx.Values.memcachedExporter.resources | nindent 12 }} + securityContext: + {{- toYaml $.ctx.Values.memcachedExporter.containerSecurityContext | nindent 12 }} + {{- if .extraVolumeMounts }} + volumeMounts: + {{- toYaml .extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + {{- with .persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .persistence.storageSize | quote }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} + diff --git a/opencloud/charts/loki/templates/memcached/_memcached-svc.tpl b/opencloud/charts/loki/templates/memcached/_memcached-svc.tpl new file mode 100644 index 0000000..8574151 --- /dev/null +++ b/opencloud/charts/loki/templates/memcached/_memcached-svc.tpl @@ -0,0 +1,42 @@ +{{/* +memcached Service +Params: + ctx = . context + valuesSection = name of the section in values.yaml + component = name of the component +valuesSection and component are specified separately because helm prefers camelcase for naming convetion and k8s components are named with snake case. +*/}} +{{- define "loki.memcached.service" -}} +{{ with (index $.ctx.Values $.valuesSection) }} +{{- if .enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.resourceName" (dict "ctx" $.ctx "component" $.component) }} + labels: + {{- include "loki.labels" $.ctx | nindent 4 }} + app.kubernetes.io/component: "memcached-{{ $.component }}" + {{- with .service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .service.annotations | nindent 4 }} + namespace: {{ $.ctx.Release.Namespace | quote }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: memcached-client + port: {{ .port }} + targetPort: {{ .port }} + {{ if $.ctx.Values.memcachedExporter.enabled -}} + - name: http-metrics + port: 9150 + targetPort: 9150 + {{ end }} + selector: + {{- include "loki.selectorLabels" $.ctx | nindent 4 }} + app.kubernetes.io/component: "memcached-{{ $.component }}" +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/monitoring/_helpers-monitoring.tpl b/opencloud/charts/loki/templates/monitoring/_helpers-monitoring.tpl new file mode 100644 index 0000000..cb693e4 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/_helpers-monitoring.tpl @@ -0,0 +1,47 @@ +{{/* +Client definition for LogsInstance +*/}} +{{- define "loki.logsInstanceClient" -}} +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- $url := printf "http://%s.%s.svc.%s:%s/loki/api/v1/push" (include "loki.writeFullname" .) .Release.Namespace .Values.global.clusterDomain ( .Values.loki.server.http_listen_port | toString ) }} +{{- if $isSingleBinary }} + {{- $url = printf "http://%s.%s.svc.%s:%s/loki/api/v1/push" (include "loki.singleBinaryFullname" .) .Release.Namespace .Values.global.clusterDomain ( .Values.loki.server.http_listen_port | toString ) }} +{{- else if .Values.gateway.enabled -}} + {{- $url = printf "http://%s.%s.svc.%s/loki/api/v1/push" (include "loki.gatewayFullname" .) .Release.Namespace .Values.global.clusterDomain }} +{{- end -}} +- url: {{ $url }} + externalLabels: + cluster: {{ include "loki.clusterLabel" . }} + {{- if .Values.enterprise.enabled }} + basicAuth: + username: + name: {{ include "enterprise-logs.selfMonitoringTenantSecret" . }} + key: username + password: + name: {{ include "enterprise-logs.selfMonitoringTenantSecret" . }} + key: password + {{- else if .Values.loki.auth_enabled }} + tenantId: {{ .Values.monitoring.selfMonitoring.tenant.name | quote }} + {{- end }} +{{- end -}} + +{{/* +Convert a recording rule group to yaml +*/}} +{{- define "loki.ruleGroupToYaml" -}} +{{- range . }} +- name: {{ .name }} + rules: + {{- toYaml .rules | nindent 4 }} +{{- end }} +{{- end }} + +{{/* +GrafanaAgent priority class name +*/}} +{{- define "grafana-agent.priorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.monitoring.selfMonitoring.grafanaAgent.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/monitoring/dashboards/_helpers-dashboards.tpl b/opencloud/charts/loki/templates/monitoring/dashboards/_helpers-dashboards.tpl new file mode 100644 index 0000000..00fd722 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/dashboards/_helpers-dashboards.tpl @@ -0,0 +1,6 @@ +{{/* +dashboards name +*/}} +{{- define "loki.dashboardsName" -}} +{{ include "loki.name" . }}-dashboards +{{- end }} diff --git a/opencloud/charts/loki/templates/monitoring/dashboards/configmap-1.yaml b/opencloud/charts/loki/templates/monitoring/dashboards/configmap-1.yaml new file mode 100644 index 0000000..6352f25 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/dashboards/configmap-1.yaml @@ -0,0 +1,30 @@ +{{- with .Values.monitoring.dashboards }} +{{- if .enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loki.dashboardsName" $ }}-1 + namespace: {{ .namespace | default $.Release.Namespace }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + "loki-chunks.json": | + {{ $.Files.Get "src/dashboards/loki-chunks.json" | fromJson | toJson }} + "loki-deletion.json": | + {{ $.Files.Get "src/dashboards/loki-deletion.json" | fromJson | toJson }} + "loki-logs.json": | + {{ $.Files.Get "src/dashboards/loki-logs.json" | fromJson | toJson }} + "loki-mixin-recording-rules.json": | + {{ $.Files.Get "src/dashboards/loki-mixin-recording-rules.json" | fromJson | toJson }} + "loki-operational.json": | + {{ $.Files.Get "src/dashboards/loki-operational.json" | fromJson | toJson }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/monitoring/dashboards/configmap-2.yaml b/opencloud/charts/loki/templates/monitoring/dashboards/configmap-2.yaml new file mode 100644 index 0000000..67d3cf4 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/dashboards/configmap-2.yaml @@ -0,0 +1,30 @@ +{{- with .Values.monitoring.dashboards }} +{{- if .enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loki.dashboardsName" $ }}-2 + namespace: {{ .namespace | default $.Release.Namespace }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + "loki-reads-resources.json": | + {{ $.Files.Get "src/dashboards/loki-reads-resources.json" | fromJson | toJson }} + "loki-reads.json": | + {{ $.Files.Get "src/dashboards/loki-reads.json" | fromJson | toJson }} + "loki-retention.json": | + {{ $.Files.Get "src/dashboards/loki-retention.json" | fromJson | toJson }} + "loki-writes-resources.json": | + {{ $.Files.Get "src/dashboards/loki-writes-resources.json" | fromJson | toJson }} + "loki-writes.json": | + {{ $.Files.Get "src/dashboards/loki-writes.json" | fromJson | toJson }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/monitoring/grafana-agent.yaml b/opencloud/charts/loki/templates/monitoring/grafana-agent.yaml new file mode 100644 index 0000000..a047e5f --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/grafana-agent.yaml @@ -0,0 +1,100 @@ +{{- if .Values.monitoring.selfMonitoring.enabled }} +{{- with .Values.monitoring.selfMonitoring.grafanaAgent }} +apiVersion: monitoring.grafana.com/v1alpha1 +kind: GrafanaAgent +metadata: + name: {{ include "loki.fullname" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + serviceAccountName: {{ include "loki.fullname" $ }}-grafana-agent + enableConfigReadAPI: {{ .enableConfigReadAPI }} + {{- include "grafana-agent.priorityClassName" $ | nindent 2 }} + logs: + instanceSelector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 8 }} + {{- with $.Values.monitoring.serviceMonitor}} + {{- if .metricsInstance.remoteWrite}} + metrics: + instanceSelector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 8 }} + {{- end }} + {{- end }} + {{- with .resources }} + resources: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "loki.fullname" $ }}-grafana-agent + namespace: {{ .namespace | default $.Release.Namespace }} + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "loki.fullname" $ }}-grafana-agent +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - events + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- nonResourceURLs: + - /metrics + - /metrics/cadvisor + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "loki.fullname" $ }}-grafana-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "loki.fullname" $ }}-grafana-agent +subjects: +- kind: ServiceAccount + name: {{ include "loki.fullname" $ }}-grafana-agent + namespace: {{ .namespace | default $.Release.Namespace }} +{{- end}} +{{- end}} diff --git a/opencloud/charts/loki/templates/monitoring/logs-instance.yaml b/opencloud/charts/loki/templates/monitoring/logs-instance.yaml new file mode 100644 index 0000000..5ae1917 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/logs-instance.yaml @@ -0,0 +1,30 @@ +{{- if .Values.monitoring.selfMonitoring.enabled }} +{{- with .Values.monitoring.selfMonitoring.logsInstance }} +apiVersion: monitoring.grafana.com/v1alpha1 +kind: LogsInstance +metadata: + name: {{ include "loki.fullname" $ }} + namespace: {{ $.Release.Namespace }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clients: + {{- include "loki.logsInstanceClient" $ | nindent 4}} + {{- with .clients}} + {{- toYaml . | nindent 4 }} + {{- end }} + + podLogsNamespaceSelector: {} + + podLogsSelector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 6 }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/opencloud/charts/loki/templates/monitoring/loki-alerts.yaml b/opencloud/charts/loki/templates/monitoring/loki-alerts.yaml new file mode 100644 index 0000000..f3333df --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/loki-alerts.yaml @@ -0,0 +1,22 @@ +{{- with .Values.monitoring.rules }} +{{- if and ($.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") .enabled .alerting }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "loki.fullname" $ }}-loki-alerts + namespace: {{ .namespace | default $.Release.Namespace }} +spec: + groups: + {{- include "loki.ruleGroupToYaml" (tpl ($.Files.Get "src/alerts.yaml.tpl") $ | fromYaml).groups | indent 4 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/monitoring/loki-rules.yaml b/opencloud/charts/loki/templates/monitoring/loki-rules.yaml new file mode 100644 index 0000000..f9eb392 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/loki-rules.yaml @@ -0,0 +1,23 @@ +{{- with .Values.monitoring.rules }} +{{- if and ($.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") .enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "loki.fullname" $ }}-loki-rules + namespace: {{ .namespace | default $.Release.Namespace }} +spec: + groups: + {{- include "loki.ruleGroupToYaml" (tpl ($.Files.Get "src/rules.yaml.tpl") $ | fromYaml).groups | indent 4 }} + {{- include "loki.ruleGroupToYaml" .additionalGroups | indent 4 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/monitoring/metrics-instance.yaml b/opencloud/charts/loki/templates/monitoring/metrics-instance.yaml new file mode 100644 index 0000000..82102c0 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/metrics-instance.yaml @@ -0,0 +1,30 @@ +{{- if .Values.monitoring.serviceMonitor.enabled }} +{{- with .Values.monitoring.serviceMonitor.metricsInstance }} +{{- if and ($.Capabilities.APIVersions.Has "monitoring.grafana.com/v1alpha1/MetricsInstance") .enabled }} +apiVersion: monitoring.grafana.com/v1alpha1 +kind: MetricsInstance +metadata: + name: {{ include "loki.fullname" $ }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .remoteWrite}} + remoteWrite: + {{- toYaml . | nindent 4 }} + {{- end }} + + serviceMonitorNamespaceSelector: {} + + serviceMonitorSelector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 6 }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/monitoring/pod-logs.yaml b/opencloud/charts/loki/templates/monitoring/pod-logs.yaml new file mode 100644 index 0000000..317339d --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/pod-logs.yaml @@ -0,0 +1,62 @@ +--- +{{- if .Values.monitoring.selfMonitoring.enabled }} +{{- with .Values.monitoring.selfMonitoring.podLogs }} +apiVersion: {{ .apiVersion }} +kind: PodLogs +metadata: + name: {{ include "loki.fullname" $ }} + namespace: {{ $.Release.Namespace }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + pipelineStages: + - cri: { } + {{- with .additionalPipelineStages }} + {{- toYaml . | nindent 4 }} + {{- end }} + relabelings: + - action: replace + sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: __host__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + replacement: "$1" + separator: "-" + sourceLabels: + - __meta_kubernetes_pod_label_app_kubernetes_io_name + - __meta_kubernetes_pod_label_app_kubernetes_io_component + targetLabel: __service__ + - action: replace + replacement: "$1" + separator: "/" + sourceLabels: + - __meta_kubernetes_namespace + - __service__ + targetLabel: job + - action: replace + sourceLabels: + - __meta_kubernetes_pod_container_name + targetLabel: container + - action: replace + replacement: "{{ include "loki.clusterLabel" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 4 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ $.Release.Namespace }} + selector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 6 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/monitoring/servicemonitor.yaml b/opencloud/charts/loki/templates/monitoring/servicemonitor.yaml new file mode 100644 index 0000000..856cee8 --- /dev/null +++ b/opencloud/charts/loki/templates/monitoring/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- with .Values.monitoring.serviceMonitor }} +{{- if and ($.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") .enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "loki.fullname" $ }} + namespace: {{ $.Release.Namespace }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "loki.selectorLabels" $ | nindent 6 }} + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + endpoints: + - port: http-metrics + path: /metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + action: replace + replacement: "{{ $.Release.Namespace }}/$1" + targetLabel: job + - action: replace + replacement: "{{ include "loki.clusterLabel" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/networkpolicy.yaml b/opencloud/charts/loki/templates/networkpolicy.yaml new file mode 100644 index 0000000..9286edb --- /dev/null +++ b/opencloud/charts/loki/templates/networkpolicy.yaml @@ -0,0 +1,203 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "kubernetes") }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-namespace-only + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Ingress + - Egress + podSelector: {} + egress: + - to: + - podSelector: {} + ingress: + - from: + - podSelector: {} + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-dns + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Egress + podSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - ports: + - port: dns + protocol: UDP + to: + - namespaceSelector: {} + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-ingress + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Ingress + podSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + {{- if .Values.gateway.enabled }} + - gateway + {{- else }} + - read + - write + {{- end }} + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + ingress: + - ports: + - port: http-metrics + protocol: TCP + {{- if .Values.networkPolicy.ingress.namespaceSelector }} + from: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.ingress.namespaceSelector | nindent 12 }} + {{- if .Values.networkPolicy.ingress.podSelector }} + podSelector: + {{- toYaml .Values.networkPolicy.ingress.podSelector | nindent 12 }} + {{- end }} + {{- end }} + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-ingress-metrics + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Ingress + podSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + ingress: + - ports: + - port: http-metrics + protocol: TCP + {{- if .Values.networkPolicy.metrics.cidrs }} + from: + {{- range $cidr := .Values.networkPolicy.metrics.cidrs }} + - ipBlock: + cidr: {{ $cidr }} + {{- end }} + {{- if .Values.networkPolicy.metrics.namespaceSelector }} + - namespaceSelector: + {{- toYaml .Values.networkPolicy.metrics.namespaceSelector | nindent 12 }} + {{- if .Values.networkPolicy.metrics.podSelector }} + podSelector: + {{- toYaml .Values.networkPolicy.metrics.podSelector | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-alertmanager + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Egress + podSelector: + matchLabels: + {{- include "loki.backendSelectorLabels" . | nindent 6 }} + egress: + - ports: + - port: {{ .Values.networkPolicy.alertmanager.port }} + protocol: TCP + {{- if .Values.networkPolicy.alertmanager.namespaceSelector }} + to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.alertmanager.namespaceSelector | nindent 12 }} + {{- if .Values.networkPolicy.alertmanager.podSelector }} + podSelector: + {{- toYaml .Values.networkPolicy.alertmanager.podSelector | nindent 12 }} + {{- end }} + {{- end }} + +{{- if .Values.networkPolicy.externalStorage.ports }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-external-storage + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Egress + podSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - ports: + {{- range $port := .Values.networkPolicy.externalStorage.ports }} + - port: {{ $port }} + protocol: TCP + {{- end }} + {{- if .Values.networkPolicy.externalStorage.cidrs }} + to: + {{- range $cidr := .Values.networkPolicy.externalStorage.cidrs }} + - ipBlock: + cidr: {{ $cidr }} + {{- end }} + {{- end }} +{{- end }} + +{{- end }} + +{{- if and .Values.networkPolicy.discovery.port (eq .Values.networkPolicy.flavor "kubernetes") }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "loki.name" . }}-egress-discovery + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + policyTypes: + - Egress + podSelector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + egress: + - ports: + - port: {{ .Values.networkPolicy.discovery.port }} + protocol: TCP + {{- if .Values.networkPolicy.discovery.namespaceSelector }} + to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.discovery.namespaceSelector | nindent 12 }} + {{- if .Values.networkPolicy.discovery.podSelector }} + podSelector: + {{- toYaml .Values.networkPolicy.discovery.podSelector | nindent 12 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl b/opencloud/charts/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl new file mode 100644 index 0000000..5477214 --- /dev/null +++ b/opencloud/charts/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl @@ -0,0 +1,58 @@ +{{/* +pattern ingester fullname +*/}} +{{- define "loki.patternIngesterFullname" -}} +{{ include "loki.fullname" . }}-pattern-ingester +{{- end }} + +{{/* +pattern ingester common labels +*/}} +{{- define "loki.patternIngesterLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: pattern-ingester +{{- end }} + +{{/* +pattern ingester selector labels +*/}} +{{- define "loki.patternIngesterSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: pattern-ingester +{{- end }} + +{{/* +pattern ingester readinessProbe +*/}} +{{- define "loki.patternIngester.readinessProbe" -}} +{{- with .Values.patternIngester.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +pattern ingester priority class name +*/}} +{{- define "loki.patternIngesterPriorityClassName" }} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.patternIngester.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* +Create the name of the pattern ingester service account +*/}} +{{- define "loki.patternIngesterServiceAccountName" -}} +{{- if .Values.patternIngester.serviceAccount.create -}} + {{ default (print (include "loki.serviceAccountName" .) "-pattern-ingester") .Values.patternIngester.serviceAccount.name }} +{{- else -}} + {{ default (include "loki.serviceAccountName" .) .Values.patternIngester.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml b/opencloud/charts/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml new file mode 100644 index 0000000..fdd2a74 --- /dev/null +++ b/opencloud/charts/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml @@ -0,0 +1,187 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +{{- if (gt (int .Values.patternIngester.replicas) 0) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.patternIngesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.patternIngesterLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.patternIngester.replicas }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.patternIngesterFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.patternIngester.persistence.enableStatefulSetAutoDeletePVC) }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.patternIngester.persistence.whenDeleted }} + whenScaled: {{ .Values.patternIngester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.patternIngesterSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.patternIngesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.patternIngesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.patternIngester.terminationGracePeriodSeconds }} + {{- with .Values.patternIngester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: pattern-ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.patternIngester.command }} + command: + - {{ coalesce .Values.patternIngester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=pattern-ingester + {{- with .Values.patternIngester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.patternIngester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.patternIngester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.patternIngester.readinessProbe" . | nindent 10 }} + volumeMounts: + - name: temp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.patternIngester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.patternIngester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.patternIngester.extraContainers }} + {{- toYaml .Values.patternIngester.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.patternIngester.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: temp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if not .Values.patternIngester.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.patternIngester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.patternIngester.persistence.enabled }} + volumeClaimTemplates: + {{- range .Values.patternIngester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/opencloud/charts/loki/templates/podsecuritypolicy.yaml b/opencloud/charts/loki/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000..05470d9 --- /dev/null +++ b/opencloud/charts/loki/templates/podsecuritypolicy.yaml @@ -0,0 +1,41 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "loki.name" . }} + labels: + {{- include "loki.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'configMap' + - 'emptyDir' + - 'persistentVolumeClaim' + - 'secret' + - 'projected' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL +{{- end }} diff --git a/opencloud/charts/loki/templates/provisioner/_helpers.yaml b/opencloud/charts/loki/templates/provisioner/_helpers.yaml new file mode 100644 index 0000000..8b04b07 --- /dev/null +++ b/opencloud/charts/loki/templates/provisioner/_helpers.yaml @@ -0,0 +1,32 @@ +{{/* +provisioner fullname +*/}} +{{- define "enterprise-logs.provisionerFullname" -}} +{{ include "loki.name" . }}-provisioner +{{- end }} + +{{/* +provisioner common labels +*/}} +{{- define "enterprise-logs.provisionerLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: provisioner +{{- end }} + +{{/* +provisioner selector labels +*/}} +{{- define "enterprise-logs.provisionerSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: provisioner +{{- end }} + +{{/* +provisioner image name +*/}} +{{- define "enterprise-logs.provisionerImage" -}} +{{- $dict := dict "service" .Values.enterprise.provisioner.image "global" .Values.global.image "defaultVersion" "latest" -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + + diff --git a/opencloud/charts/loki/templates/provisioner/job-provisioner.yaml b/opencloud/charts/loki/templates/provisioner/job-provisioner.yaml new file mode 100644 index 0000000..5a6bc06 --- /dev/null +++ b/opencloud/charts/loki/templates/provisioner/job-provisioner.yaml @@ -0,0 +1,147 @@ +{{ if and .Values.enterprise.provisioner.enabled .Values.enterprise.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "enterprise-logs.provisionerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.provisionerLabels" . | nindent 4 }} + {{- with .Values.enterprise.provisioner.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.provisioner.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install + "helm.sh/hook-weight": "15" +spec: + backoffLimit: 6 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + {{- include "enterprise-logs.provisionerSelectorLabels" . | nindent 8 }} + {{- with .Values.enterprise.provisioner.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterprise.provisioner.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.enterprise.provisioner.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + securityContext: + {{- toYaml .Values.enterprise.provisioner.securityContext | nindent 8 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: provisioner + image: {{ template "enterprise-logs.provisionerImage" . }} + imagePullPolicy: {{ .Values.enterprise.provisioner.image.pullPolicy }} + command: + - /bin/sh + - -exuc + - | + {{- range .Values.enterprise.provisioner.additionalTenants }} + /usr/bin/enterprise-logs-provisioner \ + -bootstrap-path=/bootstrap \ + -cluster-name={{ include "loki.clusterName" $ }} \ + -gel-url={{ include "loki.address" $ }} \ + -instance={{ .name }} \ + -access-policy=write-{{ .name }}:{{ .name }}:logs:write \ + -access-policy=read-{{ .name }}:{{ .name }}:logs:read \ + -token=write-{{ .name }} \ + -token=read-{{ .name }} + {{- end -}} + + {{- with .Values.monitoring.selfMonitoring.tenant }} + /usr/bin/enterprise-logs-provisioner \ + -bootstrap-path=/bootstrap \ + -cluster-name={{ include "loki.clusterName" $ }} \ + -gel-url={{ include "loki.address" $ }} \ + -instance={{ .name }} \ + -access-policy=self-monitoring:{{ .name }}:logs:write,logs:read \ + -token=self-monitoring + {{- end }} + volumeMounts: + {{- with .Values.enterprise.provisioner.extraVolumeMounts }} + {{ toYaml . | nindent 12 }} + {{- end }} + - name: bootstrap + mountPath: /bootstrap + - name: admin-token + mountPath: /bootstrap/token + subPath: token + {{- with .Values.enterprise.provisioner.env }} + env: + {{ toYaml . | nindent 12 }} + {{- end }} + containers: + - name: create-secret + image: {{ include "loki.kubectlImage" . }} + imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }} + command: + - /bin/bash + - -exuc + - | + # In case, the admin resources have already been created, the provisioner job + # does not write the token files to the bootstrap mount. + # Therefore, secrets are only created if the respective token files exist. + # Note: the following bash commands should always return a success status code. + # Therefore, in case the token file does not exist, the first clause of the + # or-operation is successful. + {{- range .Values.enterprise.provisioner.additionalTenants }} + ! test -s /bootstrap/token-write-{{ .name }} || \ + kubectl --namespace "{{ .secretNamespace }}" create secret generic "{{ include "enterprise-logs.provisionedSecretPrefix" $ }}-{{ .name }}" \ + --from-literal=token-write="$(cat /bootstrap/token-write-{{ .name }})" \ + --from-literal=token-read="$(cat /bootstrap/token-read-{{ .name }})" + {{- end }} + {{- $namespace := $.Release.Namespace }} + {{- with .Values.monitoring.selfMonitoring.tenant }} + {{- $secretNamespace := tpl .secretNamespace $ }} + ! test -s /bootstrap/token-self-monitoring || \ + kubectl --namespace "{{ $namespace }}" create secret generic "{{ include "enterprise-logs.selfMonitoringTenantSecret" $ }}" \ + --from-literal=username="{{ .name }}" \ + --from-literal=password="$(cat /bootstrap/token-self-monitoring)" + {{- if not (eq $secretNamespace $namespace) }} + ! test -s /bootstrap/token-self-monitoring || \ + kubectl --namespace "{{ $secretNamespace }}" create secret generic "{{ include "enterprise-logs.selfMonitoringTenantSecret" $ }}" \ + --from-literal=username="{{ .name }}" \ + --from-literal=password="$(cat /bootstrap/token-self-monitoring)" + {{- end }} + {{- end }} + volumeMounts: + {{- with .Values.enterprise.provisioner.extraVolumeMounts }} + {{ toYaml . | nindent 12 }} + {{- end }} + - name: bootstrap + mountPath: /bootstrap + {{- with .Values.enterprise.provisioner.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterprise.provisioner.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterprise.provisioner.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + serviceAccount: {{ include "enterprise-logs.provisionerFullname" . }} + serviceAccountName: {{ include "enterprise-logs.provisionerFullname" . }} + volumes: + - name: admin-token + secret: + secretName: "{{ include "enterprise-logs.adminTokenSecret" . }}" + - name: bootstrap + emptyDir: {} +{{- end }} diff --git a/opencloud/charts/loki/templates/provisioner/role-provisioner.yaml b/opencloud/charts/loki/templates/provisioner/role-provisioner.yaml new file mode 100644 index 0000000..1335b0f --- /dev/null +++ b/opencloud/charts/loki/templates/provisioner/role-provisioner.yaml @@ -0,0 +1,21 @@ +{{ if and (and .Values.enterprise.provisioner.enabled .Values.enterprise.enabled) (not .Values.rbac.namespaced)}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "enterprise-logs.provisionerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.provisionerLabels" . | nindent 4 }} + {{- with .Values.enterprise.provisioner.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.provisioner.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +{{- end }} diff --git a/opencloud/charts/loki/templates/provisioner/rolebinding-provisioner.yaml b/opencloud/charts/loki/templates/provisioner/rolebinding-provisioner.yaml new file mode 100644 index 0000000..d87874d --- /dev/null +++ b/opencloud/charts/loki/templates/provisioner/rolebinding-provisioner.yaml @@ -0,0 +1,26 @@ +{{ if and (and .Values.enterprise.provisioner.enabled .Values.enterprise.enabled) (not .Values.rbac.namespaced)}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "enterprise-logs.provisionerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.provisionerLabels" . | nindent 4 }} + {{- with .Values.enterprise.provisioner.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.provisioner.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "enterprise-logs.provisionerFullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "enterprise-logs.provisionerFullname" . }} + namespace: {{ $.Release.Namespace }} +{{- end }} diff --git a/opencloud/charts/loki/templates/provisioner/serviceaccount-provisioner.yaml b/opencloud/charts/loki/templates/provisioner/serviceaccount-provisioner.yaml new file mode 100644 index 0000000..81e92e9 --- /dev/null +++ b/opencloud/charts/loki/templates/provisioner/serviceaccount-provisioner.yaml @@ -0,0 +1,18 @@ +{{ if and .Values.enterprise.provisioner.enabled .Values.enterprise.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "enterprise-logs.provisionerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.provisionerLabels" . | nindent 4 }} + {{- with .Values.enterprise.provisioner.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.provisioner.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +{{- end }} diff --git a/opencloud/charts/loki/templates/querier/_helpers-querier.tpl b/opencloud/charts/loki/templates/querier/_helpers-querier.tpl new file mode 100644 index 0000000..aa557c5 --- /dev/null +++ b/opencloud/charts/loki/templates/querier/_helpers-querier.tpl @@ -0,0 +1,32 @@ +{{/* +querier fullname +*/}} +{{- define "loki.querierFullname" -}} +{{ include "loki.fullname" . }}-querier +{{- end }} + +{{/* +querier common labels +*/}} +{{- define "loki.querierLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: querier +{{- end }} + +{{/* +querier selector labels +*/}} +{{- define "loki.querierSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: querier +{{- end }} + +{{/* +querier priority class name +*/}} +{{- define "loki.querierPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.querier.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/querier/deployment-querier.yaml b/opencloud/charts/loki/templates/querier/deployment-querier.yaml new file mode 100644 index 0000000..3165d0d --- /dev/null +++ b/opencloud/charts/loki/templates/querier/deployment-querier.yaml @@ -0,0 +1,166 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.querierFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querierLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.querier.autoscaling.enabled }} + replicas: {{ .Values.querier.replicas }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.querier.maxSurge }} + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.querierSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.querierSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.querier.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.querierPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.querier.terminationGracePeriodSeconds }} + {{- with .Values.querier.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: querier + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=querier + {{- if .Values.ingester.zoneAwareReplication.enabled }} + {{- if and (.Values.ingester.zoneAwareReplication.migration.enabled) (not .Values.ingester.zoneAwareReplication.migration.readPath) }} + - -distributor.zone-awareness-enabled=false + {{- else }} + - -distributor.zone-awareness-enabled=true + {{- end }} + {{- end }} + {{- with .Values.querier.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.querier.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.querier.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.querier.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.querier.resources | nindent 12 }} + {{- if .Values.querier.extraContainers }} + {{- toYaml .Values.querier.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.querier.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.querier.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + - name: data + emptyDir: {} + {{- with .Values.querier.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/querier/hpa.yaml b/opencloud/charts/loki/templates/querier/hpa.yaml new file mode 100644 index 0000000..08d81cb --- /dev/null +++ b/opencloud/charts/loki/templates/querier/hpa.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.querier.autoscaling.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.querierFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querierLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "loki.querierFullname" . }} + minReplicas: {{ .Values.querier.autoscaling.minReplicas }} + maxReplicas: {{ .Values.querier.autoscaling.maxReplicas }} + metrics: + {{- with .Values.querier.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.querier.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.querier.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.querier.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.querier.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.querier.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/querier/poddisruptionbudget-querier.yaml b/opencloud/charts/loki/templates/querier/poddisruptionbudget-querier.yaml new file mode 100644 index 0000000..9dff3cd --- /dev/null +++ b/opencloud/charts/loki/templates/querier/poddisruptionbudget-querier.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.querier.replicas) 1) }} +{{- if kindIs "invalid" .Values.querier.maxUnavailable }} +{{- fail "`.Values.querier.maxUnavailable` must be set when `.Values.querier.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.querierFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querierLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.querierSelectorLabels" . | nindent 6 }} + {{- with .Values.querier.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/querier/service-querier.yaml b/opencloud/charts/loki/templates/querier/service-querier.yaml new file mode 100644 index 0000000..15c9c6a --- /dev/null +++ b/opencloud/charts/loki/templates/querier/service-querier.yaml @@ -0,0 +1,36 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.querierFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querierLabels" . | nindent 4 }} + {{- with .Values.querier.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.querier.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.querier.appProtocol.grpc }} + appProtocol: {{ .Values.querier.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.querierSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/query-frontend/_helpers-query-frontend.tpl b/opencloud/charts/loki/templates/query-frontend/_helpers-query-frontend.tpl new file mode 100644 index 0000000..5aebde7 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/_helpers-query-frontend.tpl @@ -0,0 +1,32 @@ +{{/* +query-frontend fullname +*/}} +{{- define "loki.queryFrontendFullname" -}} +{{ include "loki.fullname" . }}-query-frontend +{{- end }} + +{{/* +query-frontend common labels +*/}} +{{- define "loki.queryFrontendLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: query-frontend +{{- end }} + +{{/* +query-frontend selector labels +*/}} +{{- define "loki.queryFrontendSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: query-frontend +{{- end }} + +{{/* +query-frontend priority class name +*/}} +{{- define "loki.queryFrontendPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.queryFrontend.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-frontend/deployment-query-frontend.yaml b/opencloud/charts/loki/templates/query-frontend/deployment-query-frontend.yaml new file mode 100644 index 0000000..0b73fe8 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/deployment-query-frontend.yaml @@ -0,0 +1,148 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.queryFrontendFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.queryFrontendLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if not .Values.queryFrontend.autoscaling.enabled }} + replicas: {{ .Values.queryFrontend.replicas }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.queryFrontendSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryFrontend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.queryFrontendSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryFrontend.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.queryFrontend.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryFrontend.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.queryFrontendPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.queryFrontend.terminationGracePeriodSeconds }} + containers: + - name: query-frontend + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.queryFrontend.command }} + command: + - {{ coalesce .Values.queryFrontend.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=query-frontend + {{- with .Values.queryFrontend.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.queryFrontend.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.queryFrontend.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.queryFrontend.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.queryFrontend.resources | nindent 12 }} + {{- if .Values.queryFrontend.extraContainers }} + {{- toYaml .Values.queryFrontend.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.queryFrontend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryFrontend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryFrontend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.queryFrontend.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/query-frontend/hpa.yaml b/opencloud/charts/loki/templates/query-frontend/hpa.yaml new file mode 100644 index 0000000..c326287 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/hpa.yaml @@ -0,0 +1,55 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.queryFrontend.autoscaling.enabled }} +{{- $apiVersion := include "loki.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.queryFrontendFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.queryFrontendLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "loki.queryFrontendFullname" . }} + minReplicas: {{ .Values.queryFrontend.autoscaling.minReplicas }} + maxReplicas: {{ .Values.queryFrontend.autoscaling.maxReplicas }} + metrics: + {{- with .Values.queryFrontend.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.queryFrontend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.queryFrontend.autoscaling.customMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.queryFrontend.autoscaling.behavior.enabled }} + behavior: + {{- with .Values.queryFrontend.autoscaling.behavior.scaleDown }} + scaleDown: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.queryFrontend.autoscaling.behavior.scaleUp }} + scaleUp: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml b/opencloud/charts/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml new file mode 100644 index 0000000..f100405 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.queryFrontend.replicas) 1) }} +{{- if kindIs "invalid" .Values.queryFrontend.maxUnavailable }} +{{- fail "`.Values.queryFrontend.maxUnavailable` must be set when `.Values.queryFrontend.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.queryFrontendFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.queryFrontendLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.queryFrontendSelectorLabels" . | nindent 6 }} + {{- with .Values.queryFrontend.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-frontend/service-query-frontend-headless.yaml b/opencloud/charts/loki/templates/query-frontend/service-query-frontend-headless.yaml new file mode 100644 index 0000000..8da9054 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/service-query-frontend-headless.yaml @@ -0,0 +1,46 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.queryFrontendFullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.queryFrontendLabels" . | nindent 4 }} + {{- with .Values.queryFrontend.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.queryFrontend.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + type: ClusterIP + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.queryFrontend.appProtocol.grpc }} + appProtocol: {{ .Values.queryFrontend.appProtocol.grpc }} + {{- end }} + - name: grpclb + port: 9096 + targetPort: grpc + protocol: TCP + {{- if .Values.queryFrontend.appProtocol.grpc }} + appProtocol: {{ .Values.queryFrontend.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.queryFrontendSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/query-frontend/service-query-frontend.yaml b/opencloud/charts/loki/templates/query-frontend/service-query-frontend.yaml new file mode 100644 index 0000000..a239695 --- /dev/null +++ b/opencloud/charts/loki/templates/query-frontend/service-query-frontend.yaml @@ -0,0 +1,44 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.queryFrontendFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.queryFrontendLabels" . | nindent 4 }} + {{- with .Values.queryFrontend.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.queryFrontend.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- if .Values.queryFrontend.appProtocol.grpc }} + appProtocol: {{ .Values.queryFrontend.appProtocol.grpc }} + {{- end }} + - name: grpclb + port: 9096 + targetPort: grpc + protocol: TCP + {{- if .Values.queryFrontend.appProtocol.grpc }} + appProtocol: {{ .Values.queryFrontend.appProtocol.grpc }} + {{- end }} + selector: + {{- include "loki.queryFrontendSelectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/opencloud/charts/loki/templates/query-scheduler/_helpers-query-scheduler.tpl b/opencloud/charts/loki/templates/query-scheduler/_helpers-query-scheduler.tpl new file mode 100644 index 0000000..1f64802 --- /dev/null +++ b/opencloud/charts/loki/templates/query-scheduler/_helpers-query-scheduler.tpl @@ -0,0 +1,40 @@ +{{/* +query-scheduler fullname +*/}} +{{- define "loki.querySchedulerFullname" -}} +{{ include "loki.fullname" . }}-query-scheduler +{{- end }} + +{{/* +query-scheduler common labels +*/}} +{{- define "loki.querySchedulerLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: query-scheduler +{{- end }} + +{{/* +query-scheduler selector labels +*/}} +{{- define "loki.querySchedulerSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: query-scheduler +{{- end }} + +{{/* +query-scheduler image +*/}} +{{- define "loki.querySchedulerImage" -}} +{{- $dict := dict "loki" .Values.loki.image "service" .Values.queryScheduler.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.lokiImage" $dict -}} +{{- end }} + +{{/* +query-scheduler priority class name +*/}} +{{- define "loki.querySchedulerPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.queryScheduler.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-scheduler/deployment-query-scheduler.yaml b/opencloud/charts/loki/templates/query-scheduler/deployment-query-scheduler.yaml new file mode 100644 index 0000000..fbb1f28 --- /dev/null +++ b/opencloud/charts/loki/templates/query-scheduler/deployment-query-scheduler.yaml @@ -0,0 +1,146 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.querySchedulerFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querySchedulerLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.queryScheduler.replicas }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.querySchedulerSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryScheduler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.querySchedulerSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryScheduler.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.queryScheduler.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryScheduler.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.querySchedulerPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.queryScheduler.terminationGracePeriodSeconds }} + containers: + - name: query-scheduler + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=query-scheduler + {{- with .Values.queryScheduler.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.queryScheduler.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.queryScheduler.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.queryScheduler.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.queryScheduler.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.queryScheduler.extraContainers }} + {{- toYaml .Values.queryScheduler.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.queryScheduler.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryScheduler.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.queryScheduler.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.queryScheduler.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-scheduler/poddisruptionbudget-query-scheduler.yaml b/opencloud/charts/loki/templates/query-scheduler/poddisruptionbudget-query-scheduler.yaml new file mode 100644 index 0000000..ed8051f --- /dev/null +++ b/opencloud/charts/loki/templates/query-scheduler/poddisruptionbudget-query-scheduler.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.queryScheduler.replicas) 1) }} +{{- if kindIs "invalid" .Values.queryScheduler.maxUnavailable }} +{{- fail "`.Values.queryScheduler.maxUnavailable` must be set when `.Values.queryScheduler.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.querySchedulerFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querySchedulerLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.querySchedulerSelectorLabels" . | nindent 6 }} + {{- with .Values.queryScheduler.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/query-scheduler/service-query-scheduler.yaml b/opencloud/charts/loki/templates/query-scheduler/service-query-scheduler.yaml new file mode 100644 index 0000000..746c7bd --- /dev/null +++ b/opencloud/charts/loki/templates/query-scheduler/service-query-scheduler.yaml @@ -0,0 +1,38 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.querySchedulerFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.querySchedulerLabels" . | nindent 4 }} + {{- with .Values.queryScheduler.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.queryScheduler.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpclb + port: 9095 + targetPort: grpc + protocol: TCP + {{- with .Values.queryScheduler.appProtocol.grpc }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "loki.querySchedulerSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/_helpers-read.tpl b/opencloud/charts/loki/templates/read/_helpers-read.tpl new file mode 100644 index 0000000..d205314 --- /dev/null +++ b/opencloud/charts/loki/templates/read/_helpers-read.tpl @@ -0,0 +1,32 @@ +{{/* +read fullname +*/}} +{{- define "loki.readFullname" -}} +{{ include "loki.name" . }}-read +{{- end }} + +{{/* +read common labels +*/}} +{{- define "loki.readLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: read +{{- end }} + +{{/* +read selector labels +*/}} +{{- define "loki.readSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: read +{{- end }} + +{{/* +read priority class name +*/}} +{{- define "loki.readPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.read.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/deployment-read.yaml b/opencloud/charts/loki/templates/read/deployment-read.yaml new file mode 100644 index 0000000..f024a40 --- /dev/null +++ b/opencloud/charts/loki/templates/read/deployment-read.yaml @@ -0,0 +1,163 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.readFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + app.kubernetes.io/part-of: memberlist + {{- include "loki.readLabels" . | nindent 4 }} + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.backend.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.read.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + {{- if not .Values.read.autoscaling.enabled }} + replicas: {{ .Values.read.replicas }} + {{- end }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.readSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/part-of: memberlist + {{- include "loki.readSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.readPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.read.terminationGracePeriodSeconds }} + containers: + - name: loki + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.read.targetModule }} + - -legacy-read-mode=false + - -common.compactor-grpc-address={{ include "loki.backendFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ .Values.loki.server.grpc_listen_port }} + {{- with .Values.read.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.read.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.read.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: tmp + mountPath: /tmp + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end}} + {{- with .Values.read.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.read.resources | nindent 12 }} + {{- with .Values.read.extraContainers }} + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.read.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.read.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + - name: data + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.read.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/hpa.yaml b/opencloud/charts/loki/templates/read/hpa.yaml new file mode 100644 index 0000000..5515ecb --- /dev/null +++ b/opencloud/charts/loki/templates/read/hpa.yaml @@ -0,0 +1,55 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- $autoscalingv2 := .Capabilities.APIVersions.Has "autoscaling/v2" -}} +{{- if and $isSimpleScalable ( .Values.read.autoscaling.enabled ) }} +{{- if $autoscalingv2 }} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.readFullname" . }} + labels: + {{- include "loki.readLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 +{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }} + kind: Deployment + name: {{ include "loki.readFullname" . }} +{{- else }} + kind: StatefulSet + name: {{ include "loki.readFullname" . }} +{{- end }} + minReplicas: {{ .Values.read.autoscaling.minReplicas }} + maxReplicas: {{ .Values.read.autoscaling.maxReplicas }} + {{- with .Values.read.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.read.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.read.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/poddisruptionbudget-read.yaml b/opencloud/charts/loki/templates/read/poddisruptionbudget-read.yaml new file mode 100644 index 0000000..af4fcbf --- /dev/null +++ b/opencloud/charts/loki/templates/read/poddisruptionbudget-read.yaml @@ -0,0 +1,15 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (gt (int .Values.read.replicas) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.readFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.readLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.readSelectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} diff --git a/opencloud/charts/loki/templates/read/service-read-headless.yaml b/opencloud/charts/loki/templates/read/service-read-headless.yaml new file mode 100644 index 0000000..14ba0f6 --- /dev/null +++ b/opencloud/charts/loki/templates/read/service-read-headless.yaml @@ -0,0 +1,41 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{ if $isSimpleScalable }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.readFullname" . }}-headless + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.readSelectorLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.read.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + variant: headless + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.read.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + appProtocol: tcp + selector: + {{- include "loki.readSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/service-read.yaml b/opencloud/charts/loki/templates/read/service-read.yaml new file mode 100644 index 0000000..f4000fd --- /dev/null +++ b/opencloud/charts/loki/templates/read/service-read.yaml @@ -0,0 +1,37 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isSimpleScalable }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.readFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.readLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.read.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.read.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.readSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/read/statefulset-read.yaml b/opencloud/charts/loki/templates/read/statefulset-read.yaml new file mode 100644 index 0000000..93527f9 --- /dev/null +++ b/opencloud/charts/loki/templates/read/statefulset-read.yaml @@ -0,0 +1,200 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (.Values.read.legacyReadTarget ) }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.readFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + app.kubernetes.io/part-of: memberlist + {{- include "loki.readLabels" . | nindent 4 }} + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.read.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.read.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: +{{- if not .Values.read.autoscaling.enabled }} + {{- if eq .Values.deploymentMode "SingleBinary" }} + replicas: 0 + {{- else }} + replicas: {{ .Values.read.replicas }} + {{- end }} +{{- end }} + podManagementPolicy: {{ .Values.read.podManagementPolicy }} + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ printf "%s-headless" (include "loki.readFullname" .) }} + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.read.persistence.enableStatefulSetAutoDeletePVC) }} + {{/* + Data on the read nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + {{- end }} + selector: + matchLabels: + {{- include "loki.readSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/part-of: memberlist + {{- include "loki.readSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{ include "loki.enableServiceLinks" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.readPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.read.terminationGracePeriodSeconds }} + containers: + - name: loki + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.read.targetModule }} + {{- with .Values.read.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.read.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.read.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + {{- with .Values.read.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: tmp + mountPath: /tmp + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end}} + {{- with .Values.read.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.read.resources | nindent 12 }} + {{- with .Values.read.extraContainers }} + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.read.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.read.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.read.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.read.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + {{- with .Values.read.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.read.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.read.persistence.size | quote }} + {{- with .Values.read.persistence.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/results-cache/poddisruptionbudget-results-cache.yaml b/opencloud/charts/loki/templates/results-cache/poddisruptionbudget-results-cache.yaml new file mode 100644 index 0000000..6bc393a --- /dev/null +++ b/opencloud/charts/loki/templates/results-cache/poddisruptionbudget-results-cache.yaml @@ -0,0 +1,16 @@ +{{- if .Values.resultsCache.enabled }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.fullname" . }}-memcached-results-cache + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: memcached-results-cache +spec: + selector: + matchLabels: + {{- include "loki.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: memcached-results-cache + maxUnavailable: 1 +{{- end -}} diff --git a/opencloud/charts/loki/templates/results-cache/service-results-cache-headless.yaml b/opencloud/charts/loki/templates/results-cache/service-results-cache-headless.yaml new file mode 100644 index 0000000..ce92008 --- /dev/null +++ b/opencloud/charts/loki/templates/results-cache/service-results-cache-headless.yaml @@ -0,0 +1 @@ +{{- include "loki.memcached.service" (dict "ctx" $ "valuesSection" "resultsCache" "component" "results-cache" ) }} diff --git a/opencloud/charts/loki/templates/results-cache/statefulset-results-cache.yaml b/opencloud/charts/loki/templates/results-cache/statefulset-results-cache.yaml new file mode 100644 index 0000000..042e74e --- /dev/null +++ b/opencloud/charts/loki/templates/results-cache/statefulset-results-cache.yaml @@ -0,0 +1 @@ +{{- include "loki.memcached.statefulSet" (dict "ctx" $ "valuesSection" "resultsCache" "component" "results-cache" ) }} diff --git a/opencloud/charts/loki/templates/role.yaml b/opencloud/charts/loki/templates/role.yaml new file mode 100644 index 0000000..1e714b6 --- /dev/null +++ b/opencloud/charts/loki/templates/role.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.rbac.pspEnabled .Values.rbac.sccEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "loki.name" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +{{- if .Values.rbac.pspEnabled }} +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ include "loki.name" . }} +{{- end }} +{{- if .Values.rbac.sccEnabled }} +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - {{ include "loki.name" . }} + {{- if and .Values.rbac.namespaced .Values.sidecar.rules.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/rolebinding.yaml b/opencloud/charts/loki/templates/rolebinding.yaml new file mode 100644 index 0000000..cc0dfd2 --- /dev/null +++ b/opencloud/charts/loki/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if or .Values.rbac.pspEnabled .Values.rbac.sccEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "loki.name" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "loki.name" . }} +subjects: + - kind: ServiceAccount + name: {{ include "loki.serviceAccountName" . }} + namespace: {{ $.Release.Namespace }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ruler/_helpers-ruler.tpl b/opencloud/charts/loki/templates/ruler/_helpers-ruler.tpl new file mode 100644 index 0000000..2079e03 --- /dev/null +++ b/opencloud/charts/loki/templates/ruler/_helpers-ruler.tpl @@ -0,0 +1,47 @@ +{{/* +ruler fullname +*/}} +{{- define "loki.rulerFullname" -}} +{{ include "loki.fullname" . }}-ruler +{{- end }} + +{{/* +ruler common labels +*/}} +{{- define "loki.rulerLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: ruler +{{- end }} + +{{/* +ruler selector labels +*/}} +{{- define "loki.rulerSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: ruler +{{- end }} + +{{/* +ruler image +*/}} +{{- define "loki.rulerImage" -}} +{{- $dict := dict "loki" .Values.loki.image "service" .Values.ruler.image "global" .Values.global.image "defaultVersion" .Chart.AppVersion -}} +{{- include "loki.lokiImage" $dict -}} +{{- end }} + +{{/* +format rules dir +*/}} +{{- define "loki.rulerRulesDirName" -}} +rules-{{ . | replace "_" "-" | trimSuffix "-" | lower }} +{{- end }} + +{{/* +ruler priority class name +*/}} +{{- define "loki.rulerPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.ruler.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ruler/configmap-ruler.yaml b/opencloud/charts/loki/templates/ruler/configmap-ruler.yaml new file mode 100644 index 0000000..3a0c533 --- /dev/null +++ b/opencloud/charts/loki/templates/ruler/configmap-ruler.yaml @@ -0,0 +1,15 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +{{- range $dir, $files := .Values.ruler.directories }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loki.rulerFullname" $ }}-{{ include "loki.rulerRulesDirName" $dir }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.rulerLabels" $ | nindent 4 }} +data: + {{- toYaml $files | nindent 2}} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ruler/poddisruptionbudget-ruler.yaml b/opencloud/charts/loki/templates/ruler/poddisruptionbudget-ruler.yaml new file mode 100644 index 0000000..b9c03cf --- /dev/null +++ b/opencloud/charts/loki/templates/ruler/poddisruptionbudget-ruler.yaml @@ -0,0 +1,21 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed (gt (int .Values.ruler.replicas) 1) }} +{{- if kindIs "invalid" .Values.ruler.maxUnavailable }} +{{- fail "`.Values.ruler.maxUnavailable` must be set when `.Values.ruler.replicas` is greater than 1." }} +{{- else }} +apiVersion: {{ include "loki.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.rulerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.rulerLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.rulerSelectorLabels" . | nindent 6 }} + {{- with .Values.ruler.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ruler/service-ruler.yaml b/opencloud/charts/loki/templates/ruler/service-ruler.yaml new file mode 100644 index 0000000..8847683 --- /dev/null +++ b/opencloud/charts/loki/templates/ruler/service-ruler.yaml @@ -0,0 +1,37 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ruler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.rulerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.rulerSelectorLabels" . | nindent 4 }} + {{- with .Values.ruler.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.ruler.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + {{- with .Values.ruler.appProtocol.grpc }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "loki.rulerSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/ruler/statefulset-ruler.yaml b/opencloud/charts/loki/templates/ruler/statefulset-ruler.yaml new file mode 100644 index 0000000..87b42f2 --- /dev/null +++ b/opencloud/charts/loki/templates/ruler/statefulset-ruler.yaml @@ -0,0 +1,184 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if and $isDistributed .Values.ruler.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.rulerFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.rulerLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.ruler.replicas }} + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + serviceName: {{ include "loki.rulerFullname" . }} + selector: + matchLabels: + {{- include "loki.rulerSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.rulerSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.ruler.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.rulerPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ruler.terminationGracePeriodSeconds }} + {{- with .Values.ruler.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ruler + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=ruler + {{- with .Values.ruler.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.ruler.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ruler.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + - name: tmp + mountPath: /tmp/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- range $dir, $_ := .Values.ruler.directories }} + - name: {{ include "loki.rulerRulesDirName" $dir }} + mountPath: /etc/loki/rules/{{ $dir }} + {{- end }} + {{- with .Values.ruler.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.ruler.resources | nindent 12 }} + {{- with .Values.ruler.extraContainers }} + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.ruler.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.ruler.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- range $dir, $_ := .Values.ruler.directories }} + - name: {{ include "loki.rulerRulesDirName" $dir }} + configMap: + name: {{ include "loki.rulerFullname" $ }}-{{ include "loki.rulerRulesDirName" $dir }} + {{- end }} + - name: tmp + emptyDir: {} + {{- with .Values.ruler.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.ruler.persistence.enabled }} + - name: data + emptyDir: {} + {{- else }} + volumeClaimTemplates: + - metadata: + name: data + {{- with .Values.ruler.persistence.annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.ruler.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.ruler.persistence.size | quote }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/runtime-configmap.yaml b/opencloud/charts/loki/templates/runtime-configmap.yaml new file mode 100644 index 0000000..2f38193 --- /dev/null +++ b/opencloud/charts/loki/templates/runtime-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loki.name" . }}-runtime + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +data: + runtime-config.yaml: | + {{- tpl (toYaml .Values.loki.runtimeConfig) . | nindent 4 }} diff --git a/opencloud/charts/loki/templates/secret-license.yaml b/opencloud/charts/loki/templates/secret-license.yaml new file mode 100644 index 0000000..eaa519f --- /dev/null +++ b/opencloud/charts/loki/templates/secret-license.yaml @@ -0,0 +1,11 @@ +{{- if and (not .Values.enterprise.useExternalLicense) .Values.enterprise.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: enterprise-logs-license + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +data: + license.jwt: {{ .Values.enterprise.license.contents | b64enc }} +{{- end }} diff --git a/opencloud/charts/loki/templates/securitycontextconstraints.yaml b/opencloud/charts/loki/templates/securitycontextconstraints.yaml new file mode 100644 index 0000000..c3a604d --- /dev/null +++ b/opencloud/charts/loki/templates/securitycontextconstraints.yaml @@ -0,0 +1,40 @@ +{{- if .Values.rbac.sccEnabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ include "loki.name" . }} + labels: + {{- include "loki.labels" . | nindent 4 }} +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: [] +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +groups: [] +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: + - ALL +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +seccompProfiles: + - '*' +supplementalGroups: + type: RunAsAny +volumes: + - configMap + - downwardAPI + - emptyDir + - hostPath + - persistentVolumeClaim + - projected + - secret +{{- end }} diff --git a/opencloud/charts/loki/templates/service-memberlist.yaml b/opencloud/charts/loki/templates/service-memberlist.yaml new file mode 100644 index 0000000..3d46f23 --- /dev/null +++ b/opencloud/charts/loki/templates/service-memberlist.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.memberlist" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.memberlist.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp + port: 7946 + targetPort: http-memberlist + protocol: TCP + {{- with .Values.memberlist.service.publishNotReadyAddresses }} + publishNotReadyAddresses: {{ . }} + {{- end }} + selector: + {{- include "loki.selectorLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist diff --git a/opencloud/charts/loki/templates/serviceaccount.yaml b/opencloud/charts/loki/templates/serviceaccount.yaml new file mode 100644 index 0000000..dd89141 --- /dev/null +++ b/opencloud/charts/loki/templates/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{ if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "loki.serviceAccountName" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- with .Values.serviceAccount.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/_helpers-single-binary.tpl b/opencloud/charts/loki/templates/single-binary/_helpers-single-binary.tpl new file mode 100644 index 0000000..4ea3c6d --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/_helpers-single-binary.tpl @@ -0,0 +1,34 @@ +{{/* +singleBinary common labels +*/}} +{{- define "loki.singleBinaryLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: single-binary +{{- end }} + + +{{/* singleBinary selector labels */}} +{{- define "loki.singleBinarySelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: single-binary +{{- end }} + +{{/* +singleBinary priority class name +*/}} +{{- define "loki.singleBinaryPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.singleBinary.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* singleBinary replicas calculation */}} +{{- define "loki.singleBinaryReplicas" -}} +{{- $replicas := 1 }} +{{- $usingObjectStorage := eq (include "loki.isUsingObjectStorage" .) "true" }} +{{- if and $usingObjectStorage (gt (int .Values.singleBinary.replicas) 1)}} +{{- $replicas = int .Values.singleBinary.replicas -}} +{{- end }} +{{- printf "%d" $replicas }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/hpa.yaml b/opencloud/charts/loki/templates/single-binary/hpa.yaml new file mode 100644 index 0000000..c529f18 --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/hpa.yaml @@ -0,0 +1,51 @@ +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- $usingObjectStorage := eq (include "loki.isUsingObjectStorage" .) "true" }} +{{- $autoscalingv2 := .Capabilities.APIVersions.Has "autoscaling/v2" -}} +{{- if and $isSingleBinary $usingObjectStorage ( .Values.singleBinary.autoscaling.enabled ) }} +{{- if $autoscalingv2 }} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.singleBinaryFullname" . }} + labels: + {{- include "loki.singleBinaryLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.singleBinaryFullname" . }} + minReplicas: {{ .Values.singleBinary.autoscaling.minReplicas }} + maxReplicas: {{ .Values.singleBinary.autoscaling.maxReplicas }} + {{- with .Values.singleBinary.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.singleBinary.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.singleBinary.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/pdb.yaml b/opencloud/charts/loki/templates/single-binary/pdb.yaml new file mode 100644 index 0000000..bb1e1cc --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/pdb.yaml @@ -0,0 +1,16 @@ +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- if and .Values.podDisruptionBudget $isSingleBinary -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "loki.fullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.singleBinarySelectorLabels" . | nindent 6 }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/service-headless.yaml b/opencloud/charts/loki/templates/single-binary/service-headless.yaml new file mode 100644 index 0000000..7522240 --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/service-headless.yaml @@ -0,0 +1,35 @@ +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- if $isSingleBinary }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.name" . }}-headless + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.singleBinary.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + variant: headless + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.singleBinary.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + selector: + {{- include "loki.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/service.yaml b/opencloud/charts/loki/templates/single-binary/service.yaml new file mode 100644 index 0000000..352fcad --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/service.yaml @@ -0,0 +1,40 @@ +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- if $isSingleBinary }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.singleBinaryFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.singleBinary.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.singleBinary.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.singleBinarySelectorLabels" . | nindent 4 }} + {{- with .Values.singleBinary.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/single-binary/statefulset.yaml b/opencloud/charts/loki/templates/single-binary/statefulset.yaml new file mode 100644 index 0000000..54f3d92 --- /dev/null +++ b/opencloud/charts/loki/templates/single-binary/statefulset.yaml @@ -0,0 +1,278 @@ +{{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} +{{- if $isSingleBinary }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.singleBinaryFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.singleBinaryLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.singleBinary.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.singleBinary.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + replicas: {{ include "loki.singleBinaryReplicas" . }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.singleBinaryFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.singleBinary.persistence.enableStatefulSetAutoDeletePVC) (.Values.singleBinary.persistence.enabled) }} + {{/* + Data on the singleBinary nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + {{- end }} + selector: + matchLabels: + {{- include "loki.singleBinarySelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.singleBinarySelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{ include "loki.enableServiceLinks" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.singleBinaryPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.singleBinary.terminationGracePeriodSeconds }} + {{- if .Values.singleBinary.initContainers }} + initContainers: + {{- with .Values.singleBinary.initContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + {{- if .Values.sidecar.rules.enabled }} + - name: loki-sc-rules + {{- if .Values.sidecar.image.sha }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.image.pullPolicy }} + env: + - name: METHOD + value: {{ .Values.sidecar.rules.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.rules.label }}" + {{- if .Values.sidecar.rules.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.rules.labelValue }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.rules.folder }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.rules.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.rules.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.rules.searchNamespace | join "," }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.rules.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.rules.script }}" + {{- end }} + {{- if .Values.sidecar.rules.watchServerTimeout }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.rules.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.watchClientTimeout }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.rules.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.logLevel }} + - name: LOG_LEVEL + value: "{{ .Values.sidecar.rules.logLevel }}" + {{- end }} + {{- if .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml .Values.sidecar.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml .Values.sidecar.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.resources }} + resources: + {{- toYaml .Values.sidecar.resources | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.securityContext }} + securityContext: + {{- toYaml .Values.sidecar.securityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} + - name: loki + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.singleBinary.targetModule }} + {{- with .Values.singleBinary.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.singleBinary.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.singleBinary.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + {{- if .Values.singleBinary.persistence.enabled }} + - name: storage + mountPath: /var/loki + {{- end }} + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} + {{- with .Values.singleBinary.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.singleBinary.resources | nindent 12 }} + {{- with .Values.singleBinary.extraContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.singleBinary.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + {{- if .Values.sidecar.rules.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.rules.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end -}} + {{- end -}} + {{- with .Values.singleBinary.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.singleBinary.persistence.enabled }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: storage + {{- with .Values.singleBinary.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.singleBinary.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.singleBinary.persistence.size | quote }} + {{- with .Values.singleBinary.persistence.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/table-manager/_helpers-table-manager.tpl b/opencloud/charts/loki/templates/table-manager/_helpers-table-manager.tpl new file mode 100644 index 0000000..ff25832 --- /dev/null +++ b/opencloud/charts/loki/templates/table-manager/_helpers-table-manager.tpl @@ -0,0 +1,32 @@ +{{/* +table-manager fullname +*/}} +{{- define "loki.tableManagerFullname" -}} +{{ include "loki.fullname" . }}-table-manager +{{- end }} + +{{/* +table-manager common labels +*/}} +{{- define "loki.tableManagerLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: table-manager +{{- end }} + +{{/* +table-manager selector labels +*/}} +{{- define "loki.tableManagerSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: table-manager +{{- end }} + +{{/* +table-manager priority class name +*/}} +{{- define "loki.tableManagerPriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.tableManager.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/table-manager/deployment-table-manager.yaml b/opencloud/charts/loki/templates/table-manager/deployment-table-manager.yaml new file mode 100644 index 0000000..770629b --- /dev/null +++ b/opencloud/charts/loki/templates/table-manager/deployment-table-manager.yaml @@ -0,0 +1,126 @@ +{{- if .Values.tableManager.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loki.tableManagerFullname" . }} + labels: + {{- include "loki.tableManagerLabels" . | nindent 4 }} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.tableManager.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "loki.tableManagerSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tableManager.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.tableManagerSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tableManager.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.tableManagerPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.tableManager.terminationGracePeriodSeconds }} + containers: + - name: table-manager + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=table-manager + {{- with .Values.tableManager.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + {{- with .Values.tableManager.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tableManager.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.tableManager.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.tableManager.resources | nindent 12 }} + {{- if .Values.tableManager.extraContainers }} + {{- toYaml .Values.tableManager.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.tableManager.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tableManager.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.tableManager.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tableManager.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.tableManager.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/table-manager/service-table-manager.yaml b/opencloud/charts/loki/templates/table-manager/service-table-manager.yaml new file mode 100644 index 0000000..214cd36 --- /dev/null +++ b/opencloud/charts/loki/templates/table-manager/service-table-manager.yaml @@ -0,0 +1,36 @@ +{{- if .Values.tableManager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.fullname" . }}-table-manager + labels: + {{- include "loki.labels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.tableManager.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + app.kubernetes.io/component: table-manager + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.tableManager.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: table-manager +{{- end }} diff --git a/opencloud/charts/loki/templates/table-manager/servicemonitor-table-manager.yaml b/opencloud/charts/loki/templates/table-manager/servicemonitor-table-manager.yaml new file mode 100644 index 0000000..09aa727 --- /dev/null +++ b/opencloud/charts/loki/templates/table-manager/servicemonitor-table-manager.yaml @@ -0,0 +1,57 @@ +{{- if .Values.tableManager.enabled }} +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "loki.tableManagerFullname" $ }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "loki.tableManagerLabels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "loki.tableManagerSelectorLabels" $ | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/tests/_helpers.tpl b/opencloud/charts/loki/templates/tests/_helpers.tpl new file mode 100644 index 0000000..9ef7c15 --- /dev/null +++ b/opencloud/charts/loki/templates/tests/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* +Docker image name for loki helm test +*/}} +{{- define "loki.helmTestImage" -}} +{{- $dict := dict "service" .Values.test.image "global" .Values.global.image "defaultVersion" "latest" -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} + + +{{/* +test common labels +*/}} +{{- define "loki.helmTestLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: helm-test +{{- end }} diff --git a/opencloud/charts/loki/templates/tests/test-canary.yaml b/opencloud/charts/loki/templates/tests/test-canary.yaml new file mode 100644 index 0000000..9384865 --- /dev/null +++ b/opencloud/charts/loki/templates/tests/test-canary.yaml @@ -0,0 +1,36 @@ +{{- with .Values.test }} +{{- if $.Values.lokiCanary.enabled }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "loki.name" $ }}-helm-test" + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.helmTestLabels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": test +spec: + containers: + - name: loki-helm-test + image: {{ include "loki.helmTestImage" $ }} + env: + - name: CANARY_SERVICE_ADDRESS + value: "{{ .canaryServiceAddress }}" + - name: CANARY_PROMETHEUS_ADDRESS + value: "{{ .prometheusAddress }}" + {{- with .timeout }} + - name: CANARY_TEST_TIMEOUT + value: "{{ . }}" + {{- end }} + args: + - -test.v + restartPolicy: Never +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/tokengen/_helpers.yaml b/opencloud/charts/loki/templates/tokengen/_helpers.yaml new file mode 100644 index 0000000..3c881f2 --- /dev/null +++ b/opencloud/charts/loki/templates/tokengen/_helpers.yaml @@ -0,0 +1,22 @@ +{{/* +tokengen fullname +*/}} +{{- define "enterprise-logs.tokengenFullname" -}} +{{ include "loki.name" . }}-tokengen +{{- end }} + +{{/* +tokengen common labels +*/}} +{{- define "enterprise-logs.tokengenLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: tokengen +{{- end }} + +{{/* +tokengen selector labels +*/}} +{{- define "enterprise-logs.tokengenSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: tokengen +{{- end }} diff --git a/opencloud/charts/loki/templates/tokengen/clusterrole-tokengen.yaml b/opencloud/charts/loki/templates/tokengen/clusterrole-tokengen.yaml new file mode 100644 index 0000000..d357622 --- /dev/null +++ b/opencloud/charts/loki/templates/tokengen/clusterrole-tokengen.yaml @@ -0,0 +1,21 @@ +{{ if and .Values.enterprise.tokengen.enabled .Values.enterprise.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if not .Values.rbac.namespaced }}Cluster{{ end }}Role +metadata: + name: {{ template "enterprise-logs.tokengenFullname" . }} + labels: + {{- include "enterprise-logs.tokengenLabels" . | nindent 4 }} + {{- with .Values.enterprise.tokengen.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.tokengen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "patch"] +{{- end }} diff --git a/opencloud/charts/loki/templates/tokengen/clusterrolebinding-tokengen.yaml b/opencloud/charts/loki/templates/tokengen/clusterrolebinding-tokengen.yaml new file mode 100644 index 0000000..fb21d8f --- /dev/null +++ b/opencloud/charts/loki/templates/tokengen/clusterrolebinding-tokengen.yaml @@ -0,0 +1,25 @@ +{{ if and .Values.enterprise.tokengen.enabled .Values.enterprise.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if not .Values.rbac.namespaced }}Cluster{{ end }}RoleBinding +metadata: + name: {{ template "enterprise-logs.tokengenFullname" . }} + labels: + {{- include "enterprise-logs.tokengenLabels" . | nindent 4 }} + {{- with .Values.enterprise.tokengen.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.tokengen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ if not .Values.rbac.namespaced }}Cluster{{ end }}Role + name: {{ template "enterprise-logs.tokengenFullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "enterprise-logs.tokengenFullname" . }} + namespace: {{ $.Release.Namespace }} +{{- end }} diff --git a/opencloud/charts/loki/templates/tokengen/job-tokengen.yaml b/opencloud/charts/loki/templates/tokengen/job-tokengen.yaml new file mode 100644 index 0000000..b0950d6 --- /dev/null +++ b/opencloud/charts/loki/templates/tokengen/job-tokengen.yaml @@ -0,0 +1,143 @@ +{{ if and .Values.enterprise.tokengen.enabled .Values.enterprise.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "enterprise-logs.tokengenFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.tokengenLabels" . | nindent 4 }} + {{- with .Values.enterprise.tokengen.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.tokengen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install + "helm.sh/hook-weight": "10" +spec: + backoffLimit: 6 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + {{- include "enterprise-logs.tokengenSelectorLabels" . | nindent 8 }} + {{- with .Values.enterprise.tokengen.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.enterprise.tokengen.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.enterprise.tokengen.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + securityContext: + {{- toYaml .Values.enterprise.tokengen.securityContext | nindent 8 }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + initContainers: + - name: loki + image: {{ template "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + # The shared emptyDir exists only while the job is running, and is deleted once the job is completed. + # The tokengen generates a new admin token in case the 'token-file' file doesn't exist. + # As a result, subsequent executions of this tokengen job will generate new admin tokens. + # Note that previously generated tokens remain valid, as these remain present in the object storage. + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.enterprise.tokengen.targetModule }} + - -tokengen.token-file=/shared/admin-token + {{- with .Values.enterprise.tokengen.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.enterprise.tokengen.extraVolumeMounts }} + {{ toYaml .Values.enterprise.tokengen.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: shared + mountPath: /shared + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: license + mountPath: /etc/loki/license + env: + {{- if .Values.enterprise.tokengen.env }} + {{ toYaml .Values.enterprise.tokengen.env | nindent 12 }} + {{- end }} + {{- with .Values.enterprise.tokengen.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: create-secret + image: {{ include "loki.kubectlImage" . }} + imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }} + command: + - /bin/bash + - -euc + - | + # Create or update admin token secrets generated by tokengen job + kubectl create secret generic "{{ include "enterprise-logs.adminTokenSecret" . }}" \ + --from-file=token=/shared/admin-token \ + --dry-run=client -o yaml \ + | kubectl apply -f - + {{- with .Values.enterprise.adminToken.additionalNamespaces }} + {{- range . }} + kubectl --namespace "{{ . }}" create secret generic "{{ include "enterprise-logs.adminTokenSecret" $ }}" \ + --from-file=token=/shared/admin-token \ + --dry-run=client -o yaml \ + | kubectl apply -f - + {{- end }} + {{- end }} + volumeMounts: + {{- if .Values.enterprise.tokengen.extraVolumeMounts }} + {{ toYaml .Values.enterprise.tokengen.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: shared + mountPath: /shared + - name: config + mountPath: /etc/loki/config + - name: license + mountPath: /etc/loki/license + restartPolicy: OnFailure + serviceAccount: {{ template "enterprise-logs.tokengenFullname" . }} + serviceAccountName: {{ template "enterprise-logs.tokengenFullname" . }} + {{- with .Values.enterprise.tokengen.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterprise.tokengen.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.enterprise.tokengen.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + - name: shared + emptyDir: {} + {{- if .Values.enterprise.tokengen.extraVolumes }} + {{ toYaml .Values.enterprise.tokengen.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/tokengen/serviceaccount-tokengen.yaml b/opencloud/charts/loki/templates/tokengen/serviceaccount-tokengen.yaml new file mode 100644 index 0000000..6f0e5a3 --- /dev/null +++ b/opencloud/charts/loki/templates/tokengen/serviceaccount-tokengen.yaml @@ -0,0 +1,18 @@ +{{ if and .Values.enterprise.tokengen.enabled .Values.enterprise.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "enterprise-logs.tokengenFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "enterprise-logs.tokengenLabels" . | nindent 4 }} + {{- with .Values.enterprise.tokengen.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.enterprise.tokengen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +{{- end }} diff --git a/opencloud/charts/loki/templates/validate.yaml b/opencloud/charts/loki/templates/validate.yaml new file mode 100644 index 0000000..93e2490 --- /dev/null +++ b/opencloud/charts/loki/templates/validate.yaml @@ -0,0 +1,41 @@ +{{- if .Values.config }} +{{- fail "Top level 'config' is not allowed. Most common configuration sections are exposed under the `loki` section. If you need to override the whole config, provide the configuration as a string that can contain template expressions under `loki.config`. Alternatively, you can provide the configuration as an external secret." }} +{{- end }} + +{{- if and (not .Values.lokiCanary.enabled) .Values.test.enabled }} +{{- fail "Helm test requires the Loki Canary to be enabled"}} +{{- end }} + +{{- $singleBinaryReplicas := int .Values.singleBinary.replicas }} +{{- $isUsingFilesystem := eq (include "loki.isUsingObjectStorage" .) "false" }} +{{- $atLeastOneScalableReplica := or (gt (int .Values.backend.replicas) 0) (gt (int .Values.read.replicas) 0) (gt (int .Values.write.replicas) 0) }} +{{- $atLeastOneDistributedReplica := or (gt (int .Values.ingester.replicas) 0) (gt (int .Values.distributor.replicas) 0) (gt (int .Values.querier.replicas) 0) (gt (int .Values.queryFrontend.replicas) 0) (gt (int .Values.queryScheduler.replicas) 0) (gt (int .Values.indexGateway.replicas) 0) (gt (int .Values.compactor.replicas) 0) (gt (int .Values.ruler.replicas) 0) }} + +{{- if and $isUsingFilesystem (gt $singleBinaryReplicas 1) }} +{{- fail "Cannot run more than 1 Single Binary replica without an object storage backend."}} +{{- end }} + +{{- if and $isUsingFilesystem (and (eq $singleBinaryReplicas 0) (or $atLeastOneScalableReplica $atLeastOneDistributedReplica)) }} +{{- fail "Cannot run scalable targets (backend, read, write) or distributed targets without an object storage backend."}} +{{- end }} + +{{- if and $atLeastOneScalableReplica $atLeastOneDistributedReplica (ne .Values.deploymentMode "SimpleScalable<->Distributed") }} +{{- fail "You have more than zero replicas configured for scalable targets (backend, read, write) and distributed targets. If this was intentional change the deploymentMode to the transitional 'SimpleScalable<->Distributed' mode" }} +{{- end }} + +{{- if and (gt $singleBinaryReplicas 0) $atLeastOneDistributedReplica }} +{{- fail "You have more than zero replicas configured for both the single binary and distributed targets, there is no transition mode between these targets please change one or the other to zero or transition to the SimpleScalable mode first."}} +{{- end }} + +{{- if and (gt $singleBinaryReplicas 0) $atLeastOneScalableReplica (ne .Values.deploymentMode "SingleBinary<->SimpleScalable") }} +{{- fail "You have more than zero replicas configured for both the single binary and simple scalable targets. If this was intentional change the deploymentMode to the transitional 'SingleBinary<->SimpleScalable' mode"}} +{{- end }} + +{{- if and (or (not (empty .Values.loki.schemaConfig)) (not (empty .Values.loki.structuredConfig.schema_config))) .Values.loki.useTestSchema }} +{{- fail "loki.useTestSchema must be false if loki.schemaConfig or loki.structuredConfig.schema_config are defined."}} +{{- end }} + + +{{- if and (empty .Values.loki.schemaConfig) (empty .Values.loki.structuredConfig.schema_config) (not .Values.loki.useTestSchema) }} +{{- fail "You must provide a schema_config for Loki, one is not provided as this will be individual for every Loki cluster. See https://grafana.com/docs/loki/latest/operations/storage/schema/ for schema information. For quick testing (with no persistence) add `--set loki.useTestSchema=true`"}} +{{- end }} \ No newline at end of file diff --git a/opencloud/charts/loki/templates/write/_helpers-write.tpl b/opencloud/charts/loki/templates/write/_helpers-write.tpl new file mode 100644 index 0000000..8f526bc --- /dev/null +++ b/opencloud/charts/loki/templates/write/_helpers-write.tpl @@ -0,0 +1,32 @@ +{{/* +write fullname +*/}} +{{- define "loki.writeFullname" -}} +{{ include "loki.name" . }}-write +{{- end }} + +{{/* +write common labels +*/}} +{{- define "loki.writeLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: write +{{- end }} + +{{/* +write selector labels +*/}} +{{- define "loki.writeSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: write +{{- end }} + +{{/* +write priority class name +*/}} +{{- define "loki.writePriorityClassName" -}} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.write.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/write/hpa.yaml b/opencloud/charts/loki/templates/write/hpa.yaml new file mode 100644 index 0000000..ba88ee2 --- /dev/null +++ b/opencloud/charts/loki/templates/write/hpa.yaml @@ -0,0 +1,51 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- $autoscalingv2 := .Capabilities.APIVersions.Has "autoscaling/v2" -}} +{{- if and $isSimpleScalable ( .Values.write.autoscaling.enabled ) }} +{{- if $autoscalingv2 }} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "loki.writeFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.writeLabels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "loki.writeFullname" . }} + minReplicas: {{ .Values.write.autoscaling.minReplicas }} + maxReplicas: {{ .Values.write.autoscaling.maxReplicas }} + {{- with .Values.write.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.write.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.write.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if $autoscalingv2 }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/templates/write/poddisruptionbudget-write.yaml b/opencloud/charts/loki/templates/write/poddisruptionbudget-write.yaml new file mode 100644 index 0000000..24e1356 --- /dev/null +++ b/opencloud/charts/loki/templates/write/poddisruptionbudget-write.yaml @@ -0,0 +1,15 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if and $isSimpleScalable (gt (int .Values.write.replicas) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "loki.writeFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.writeLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "loki.writeSelectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} diff --git a/opencloud/charts/loki/templates/write/service-write-headless.yaml b/opencloud/charts/loki/templates/write/service-write-headless.yaml new file mode 100644 index 0000000..84cf5d7 --- /dev/null +++ b/opencloud/charts/loki/templates/write/service-write-headless.yaml @@ -0,0 +1,41 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isSimpleScalable }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.writeFullname" . }}-headless + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.writeSelectorLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.write.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + variant: headless + prometheus.io/service-monitor: "false" + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.write.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + appProtocol: tcp + selector: + {{- include "loki.writeSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/write/service-write.yaml b/opencloud/charts/loki/templates/write/service-write.yaml new file mode 100644 index 0000000..9603706 --- /dev/null +++ b/opencloud/charts/loki/templates/write/service-write.yaml @@ -0,0 +1,37 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isSimpleScalable }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loki.writeFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.writeLabels" . | nindent 4 }} + {{- with .Values.loki.serviceLabels }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.write.service.labels }} + {{- toYaml . | nindent 4}} + {{- end }} + annotations: + {{- with .Values.loki.serviceAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.write.service.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.loki.server.http_listen_port }} + targetPort: http-metrics + protocol: TCP + - name: grpc + port: {{ .Values.loki.server.grpc_listen_port }} + targetPort: grpc + protocol: TCP + selector: + {{- include "loki.writeSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/opencloud/charts/loki/templates/write/statefulset-write.yaml b/opencloud/charts/loki/templates/write/statefulset-write.yaml new file mode 100644 index 0000000..fc3b301 --- /dev/null +++ b/opencloud/charts/loki/templates/write/statefulset-write.yaml @@ -0,0 +1,217 @@ +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} +{{- if $isSimpleScalable }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.writeFullname" . }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "loki.writeLabels" . | nindent 4 }} + app.kubernetes.io/part-of: memberlist + {{- if or (not (empty .Values.loki.annotations)) (not (empty .Values.backend.annotations))}} + annotations: + {{- with .Values.loki.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.write.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: +{{- if not .Values.write.autoscaling.enabled }} + {{- if eq .Values.deploymentMode "SingleBinary" }} + replicas: 0 + {{- else }} + replicas: {{ .Values.write.replicas }} + {{- end }} +{{- end }} + podManagementPolicy: {{ .Values.write.podManagementPolicy }} + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.writeFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" (include "loki.kubeVersion" .)) (.Values.write.persistence.enableStatefulSetAutoDeletePVC) (.Values.write.persistence.volumeClaimsEnabled) }} + {{/* + Data on the write nodes is easy to replace, so we want to always delete PVCs to make + operation easier, and will rely on re-fetching data when needed. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + {{- end }} + selector: + matchLabels: + {{- include "loki.writeSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "loki.configMapOrSecretContentHash" (dict "ctx" . "name" "/config.yaml") }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.writeSelectorLabels" . | nindent 8 }} + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.selectorLabels }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + app.kubernetes.io/part-of: memberlist + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{ include "loki.enableServiceLinks" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.writePriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.write.terminationGracePeriodSeconds }} + {{- if .Values.write.initContainers }} + initContainers: + {{- with .Values.write.initContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: loki + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target={{ .Values.write.targetModule }} + {{- with .Values.write.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.loki.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.loki.server.grpc_listen_port }} + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.write.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.write.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + readinessProbe: + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + {{- if .Values.write.lifecycle }} + lifecycle: + {{- toYaml .Values.write.lifecycle | nindent 12 }} + {{- else if .Values.write.autoscaling.enabled }} + lifecycle: + preStop: + httpGet: + path: "/ingester/shutdown?terminate=false" + port: http-metrics + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.write.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.write.resources | nindent 12 }} + {{- with .Values.write.extraContainers }} + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.write.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.dnsConfig }} + dnsConfig: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.write.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.write.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if not .Values.write.persistence.volumeClaimsEnabled }} + - name: data + {{- toYaml .Values.write.persistence.dataVolumeParameters | nindent 10 }} + {{- end}} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- with .Values.write.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.write.persistence.volumeClaimsEnabled }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + {{- with .Values.write.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .Values.write.persistence.storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.write.persistence.size | quote }} + {{- with .Values.write.persistence.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.write.extraVolumeClaimTemplates }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/opencloud/charts/loki/test/config_test.go b/opencloud/charts/loki/test/config_test.go new file mode 100644 index 0000000..6926c7b --- /dev/null +++ b/opencloud/charts/loki/test/config_test.go @@ -0,0 +1,220 @@ +package test + +import ( + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +type replicas struct { + Replicas int `yaml:"replicas"` +} +type loki struct { + Storage struct { + Type string `yaml:"type"` + } `yaml:"storage"` +} + +type values struct { + DeploymentMode string `yaml:"deploymentMode"` + Backend replicas `yaml:"backend"` + Compactor replicas `yaml:"compactor"` + Distributor replicas `yaml:"distributor"` + IndexGateway replicas `yaml:"indexGateway"` + Ingester replicas `yaml:"ingester"` + Querier replicas `yaml:"querier"` + QueryFrontend replicas `yaml:"queryFrontend"` + QueryScheduler replicas `yaml:"queryScheduler"` + Read replicas `yaml:"read"` + Ruler replicas `yaml:"ruler"` + SingleBinary replicas `yaml:"singleBinary"` + Write replicas `yaml:"write"` + + Loki loki `yaml:"loki"` +} + +func templateConfig(t *testing.T, vals values) error { + y, err := yaml.Marshal(&vals) + require.NoError(t, err) + require.Greater(t, len(y), 0) + + f, err := os.CreateTemp("", "values.yaml") + require.NoError(t, err) + + _, err = f.Write(y) + require.NoError(t, err) + + cmd := exec.Command("helm", "dependency", "build") + // Dependency build needs to be run from the parent directory where the chart is located. + cmd.Dir = "../" + var cmdOutput []byte + if cmdOutput, err = cmd.CombinedOutput(); err != nil { + t.Log("dependency build failed", "err", string(cmdOutput)) + return err + } + + cmd = exec.Command("helm", "template", "../", "--values", f.Name()) + if cmdOutput, err := cmd.CombinedOutput(); err != nil { + t.Log("template failed", "err", string(cmdOutput)) + return err + } + + return nil +} + +// E.Welch these tests fail because the templateConfig function above can't resolve the chart dependencies and I'm not sure how to fix this.... + +//func Test_InvalidConfigs(t *testing.T) { +// t.Run("running both single binary and scalable targets", func(t *testing.T) { +// vals := values{ +// SingleBinary: replicas{Replicas: 1}, +// Write: replicas{Replicas: 1}, +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "gcs"}, +// }, +// } +// require.Error(t, templateConfig(t, vals)) +// }) +// +// t.Run("running both single binary and distributed targets", func(t *testing.T) { +// vals := values{ +// SingleBinary: replicas{Replicas: 1}, +// Distributor: replicas{Replicas: 1}, +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "gcs"}, +// }, +// } +// require.Error(t, templateConfig(t, vals)) +// }) +// +// t.Run("running both scalable and distributed targets", func(t *testing.T) { +// vals := values{ +// Read: replicas{Replicas: 1}, +// Distributor: replicas{Replicas: 1}, +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "gcs"}, +// }, +// } +// require.Error(t, templateConfig(t, vals)) +// }) +// +// t.Run("running scalable with filesystem storage", func(t *testing.T) { +// vals := values{ +// Read: replicas{Replicas: 1}, +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "filesystem"}, +// }, +// } +// +// require.Error(t, templateConfig(t, vals)) +// }) +// +// t.Run("running distributed with filesystem storage", func(t *testing.T) { +// vals := values{ +// Distributor: replicas{Replicas: 1}, +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "filesystem"}, +// }, +// } +// +// require.Error(t, templateConfig(t, vals)) +// }) +//} +// +//func Test_ValidConfigs(t *testing.T) { +// t.Run("single binary", func(t *testing.T) { +// vals := values{ +// +// DeploymentMode: "SingleBinary", +// +// SingleBinary: replicas{Replicas: 1}, +// +// Backend: replicas{Replicas: 0}, +// Compactor: replicas{Replicas: 0}, +// Distributor: replicas{Replicas: 0}, +// IndexGateway: replicas{Replicas: 0}, +// Ingester: replicas{Replicas: 0}, +// Querier: replicas{Replicas: 0}, +// QueryFrontend: replicas{Replicas: 0}, +// QueryScheduler: replicas{Replicas: 0}, +// Read: replicas{Replicas: 0}, +// Ruler: replicas{Replicas: 0}, +// Write: replicas{Replicas: 0}, +// +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "filesystem"}, +// }, +// } +// require.NoError(t, templateConfig(t, vals)) +// }) +// +// t.Run("scalable", func(t *testing.T) { +// vals := values{ +// +// DeploymentMode: "SimpleScalable", +// +// Backend: replicas{Replicas: 1}, +// Read: replicas{Replicas: 1}, +// Write: replicas{Replicas: 1}, +// +// Compactor: replicas{Replicas: 0}, +// Distributor: replicas{Replicas: 0}, +// IndexGateway: replicas{Replicas: 0}, +// Ingester: replicas{Replicas: 0}, +// Querier: replicas{Replicas: 0}, +// QueryFrontend: replicas{Replicas: 0}, +// QueryScheduler: replicas{Replicas: 0}, +// Ruler: replicas{Replicas: 0}, +// SingleBinary: replicas{Replicas: 0}, +// +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "gcs"}, +// }, +// } +// require.NoError(t, templateConfig(t, vals)) +// }) +// +// t.Run("distributed", func(t *testing.T) { +// vals := values{ +// DeploymentMode: "Distributed", +// +// Compactor: replicas{Replicas: 1}, +// Distributor: replicas{Replicas: 1}, +// IndexGateway: replicas{Replicas: 1}, +// Ingester: replicas{Replicas: 1}, +// Querier: replicas{Replicas: 1}, +// QueryFrontend: replicas{Replicas: 1}, +// QueryScheduler: replicas{Replicas: 1}, +// Ruler: replicas{Replicas: 1}, +// +// Backend: replicas{Replicas: 0}, +// Read: replicas{Replicas: 0}, +// SingleBinary: replicas{Replicas: 0}, +// Write: replicas{Replicas: 0}, +// +// Loki: loki{ +// Storage: struct { +// Type string `yaml:"type"` +// }{Type: "gcs"}, +// }, +// } +// require.NoError(t, templateConfig(t, vals)) +// }) +//} diff --git a/opencloud/charts/loki/values.yaml b/opencloud/charts/loki/values.yaml new file mode 100644 index 0000000..855e589 --- /dev/null +++ b/opencloud/charts/loki/values.yaml @@ -0,0 +1,3618 @@ +# -- Overrides the version used to determine compatibility of resources with the target Kubernetes cluster. +# This is useful when using `helm template`, because then helm will use the client version of kubectl as the Kubernetes version, +# which may or may not match your cluster's server version. Example: 'v1.24.4'. Set to null to use the version that helm +# devises. +kubeVersionOverride: null + +global: + image: + # -- Overrides the Docker registry globally for all images + registry: null + # -- Overrides the priorityClassName for all pods + priorityClassName: null + # -- configures cluster domain ("cluster.local" by default) + clusterDomain: "cluster.local" + # -- configures DNS service name + dnsService: "kube-dns" + # -- configures DNS service namespace + dnsNamespace: "kube-system" +# -- Overrides the chart's name +nameOverride: null +# -- Overrides the chart's computed fullname +fullnameOverride: null +# -- Overrides the chart's cluster label +clusterLabelOverride: null +# -- Image pull secrets for Docker images +imagePullSecrets: [] +# -- Deployment mode lets you specify how to deploy Loki. +# There are 3 options: +# - SingleBinary: Loki is deployed as a single binary, useful for small installs typically without HA, up to a few tens of GB/day. +# - SimpleScalable: Loki is deployed as 3 targets: read, write, and backend. Useful for medium installs easier to manage than distributed, up to a about 1TB/day. +# - Distributed: Loki is deployed as individual microservices. The most complicated but most capable, useful for large installs, typically over 1TB/day. +# There are also 2 additional modes used for migrating between deployment modes: +# - SingleBinary<->SimpleScalable: Migrate from SingleBinary to SimpleScalable (or vice versa) +# - SimpleScalable<->Distributed: Migrate from SimpleScalable to Distributed (or vice versa) +# Note: SimpleScalable and Distributed REQUIRE the use of object storage. +deploymentMode: SimpleScalable +###################################################################################################################### +# +# Base Loki Configs including kubernetes configurations and configurations for Loki itself, +# see below for more specifics on Loki's configuration. +# +###################################################################################################################### +# -- Configuration for running Loki +# @default -- See values.yaml +loki: + # Configures the readiness probe for all of the Loki pods + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 30 + timeoutSeconds: 1 + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/loki + # -- Overrides the image tag whose default is the chart's appVersion + tag: 3.3.1 + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Common annotations for all deployments/StatefulSets + annotations: {} + # -- Common annotations for all pods + podAnnotations: {} + # -- Common labels for all pods + podLabels: {} + # -- Common annotations for all services + serviceAnnotations: {} + # -- Common labels for all services + serviceLabels: {} + # -- The number of old ReplicaSets to retain to allow rollback + revisionHistoryLimit: 10 + # -- The SecurityContext for Loki pods + podSecurityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + # -- The SecurityContext for Loki containers + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + # -- Should enableServiceLinks be enabled. Default to enable + enableServiceLinks: true + ###################################################################################################################### + # + # Loki Configuration + # + # There are several ways to pass configuration to Loki, listing them here in order of our preference for how + # you should use this chart. + # 1. Use the templated value of loki.config below and the corresponding override sections which follow. + # This allows us to set a lot of important Loki configurations and defaults and also allows us to maintain them + # over time as Loki changes and evolves. + # 2. Use the loki.structuredConfig section. + # This will completely override the templated value of loki.config, so you MUST provide the entire Loki config + # including any configuration that we set in loki.config unless you explicitly are trying to change one of those + # values and are not able to do so with the templated sections. + # If you choose this approach the burden is on you to maintain any changes we make to the templated config. + # 3. Use an existing secret or configmap to provide the configuration. + # This option is mostly provided for folks who have external processes which provide or modify the configuration. + # When using this option you can specify a different name for loki.generatedConfigObjectName and configObjectName + # if you have a process which takes the generated config and modifies it, or you can stop the chart from generating + # a config entirely by setting loki.generatedConfigObjectName to + # + ###################################################################################################################### + + # -- Defines what kind of object stores the configuration, a ConfigMap or a Secret. + # In order to move sensitive information (such as credentials) from the ConfigMap/Secret to a more secure location (e.g. vault), it is possible to use [environment variables in the configuration](https://grafana.com/docs/loki/latest/configuration/#use-environment-variables-in-the-configuration). + # Such environment variables can be then stored in a separate Secret and injected via the global.extraEnvFrom value. For details about environment injection from a Secret please see [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-as-container-environment-variables). + configStorageType: ConfigMap + # -- The name of the object which Loki will mount as a volume containing the config. + # If the configStorageType is Secret, this will be the name of the Secret, if it is ConfigMap, this will be the name of the ConfigMap. + # The value will be passed through tpl. + configObjectName: '{{ include "loki.name" . }}' + # -- The name of the Secret or ConfigMap that will be created by this chart. + # If empty, no configmap or secret will be created. + # The value will be passed through tpl. + generatedConfigObjectName: '{{ include "loki.name" . }}' + # -- Config file contents for Loki + # @default -- See values.yaml + config: | + {{- if .Values.enterprise.enabled}} + {{- tpl .Values.enterprise.config . }} + {{- else }} + auth_enabled: {{ .Values.loki.auth_enabled }} + {{- end }} + + {{- with .Values.loki.server }} + server: + {{- toYaml . | nindent 2}} + {{- end}} + + pattern_ingester: + enabled: {{ .Values.loki.pattern_ingester.enabled }} + + memberlist: + {{- if .Values.loki.memberlistConfig }} + {{- toYaml .Values.loki.memberlistConfig | nindent 2 }} + {{- else }} + {{- if .Values.loki.extraMemberlistConfig}} + {{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}} + {{- end }} + join_members: + - {{ include "loki.memberlist" . }} + {{- with .Values.migrate.fromDistributed }} + {{- if .enabled }} + - {{ .memberlistService }} + {{- end }} + {{- end }} + {{- end }} + + {{- with .Values.loki.ingester }} + ingester: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- if .Values.loki.commonConfig}} + common: + {{- toYaml .Values.loki.commonConfig | nindent 2}} + storage: + {{- include "loki.commonStorageConfig" . | nindent 4}} + {{- end}} + + {{- with .Values.loki.limits_config }} + limits_config: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + runtime_config: + file: /etc/loki/runtime-config/runtime-config.yaml + + {{- with .Values.chunksCache }} + {{- if .enabled }} + chunk_store_config: + chunk_cache_config: + default_validity: {{ .defaultValidity }} + background: + writeback_goroutines: {{ .writebackParallelism }} + writeback_buffer: {{ .writebackBuffer }} + writeback_size_limit: {{ .writebackSizeLimit }} + memcached: + batch_size: {{ .batchSize }} + parallelism: {{ .parallelism }} + memcached_client: + addresses: dnssrvnoa+_memcached-client._tcp.{{ template "loki.fullname" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc + consistent_hash: true + timeout: {{ .timeout }} + max_idle_conns: 72 + {{- end }} + {{- end }} + + {{- if .Values.loki.schemaConfig }} + schema_config: + {{- toYaml .Values.loki.schemaConfig | nindent 2}} + {{- end }} + + {{- if .Values.loki.useTestSchema }} + schema_config: + {{- toYaml .Values.loki.testSchemaConfig | nindent 2}} + {{- end }} + + {{ include "loki.rulerConfig" . }} + + {{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }} + table_manager: + retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }} + retention_period: {{ .Values.tableManager.retention_period }} + {{- end }} + + query_range: + align_queries_with_step: true + {{- with .Values.loki.query_range }} + {{- tpl (. | toYaml) $ | nindent 2 }} + {{- end }} + {{- if .Values.resultsCache.enabled }} + {{- with .Values.resultsCache }} + cache_results: true + results_cache: + cache: + default_validity: {{ .defaultValidity }} + background: + writeback_goroutines: {{ .writebackParallelism }} + writeback_buffer: {{ .writebackBuffer }} + writeback_size_limit: {{ .writebackSizeLimit }} + memcached_client: + consistent_hash: true + addresses: dnssrvnoa+_memcached-client._tcp.{{ template "loki.fullname" $ }}-results-cache.{{ $.Release.Namespace }}.svc + timeout: {{ .timeout }} + update_interval: 1m + {{- end }} + {{- end }} + + {{- with .Values.loki.storage_config }} + storage_config: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.query_scheduler }} + query_scheduler: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.compactor }} + compactor: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.analytics }} + analytics: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.querier }} + querier: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.index_gateway }} + index_gateway: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.frontend }} + frontend: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.frontend_worker }} + frontend_worker: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.distributor }} + distributor: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + tracing: + enabled: {{ .Values.loki.tracing.enabled }} + + {{- with .Values.loki.bloom_build }} + bloom_build: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + + {{- with .Values.loki.bloom_gateway }} + bloom_gateway: + {{- tpl (. | toYaml) $ | nindent 4 }} + {{- end }} + # Should authentication be enabled + auth_enabled: true + # -- memberlist configuration (overrides embedded default) + memberlistConfig: {} + # -- Extra memberlist configuration + extraMemberlistConfig: {} + # -- Tenants list to be created on nginx htpasswd file, with name and password keys + tenants: [] + # -- Check https://grafana.com/docs/loki/latest/configuration/#server for more info on the server configuration. + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + http_server_read_timeout: 600s + http_server_write_timeout: 600s + # -- Limits config + limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + max_cache_freshness_per_query: 10m + split_queries_by_interval: 15m + query_timeout: 300s + volume_enabled: true + # -- Provides a reloadable runtime configuration file for some specific configuration + runtimeConfig: {} + # -- Check https://grafana.com/docs/loki/latest/configuration/#common_config for more info on how to provide a common configuration + commonConfig: + path_prefix: /var/loki + replication_factor: 3 + compactor_address: '{{ include "loki.compactorAddress" . }}' + # -- Storage config. Providing this will automatically populate all necessary storage configs in the templated config. + storage: + # Loki requires a bucket for chunks and the ruler. GEL requires a third bucket for the admin API. + # Please provide these values if you are using object storage. + # bucketNames: + # chunks: FIXME + # ruler: FIXME + # admin: FIXME + type: s3 + s3: + s3: null + endpoint: null + region: null + secretAccessKey: null + accessKeyId: null + signatureVersion: null + s3ForcePathStyle: false + insecure: false + http_config: {} + # -- Check https://grafana.com/docs/loki/latest/configure/#s3_storage_config for more info on how to provide a backoff_config + backoff_config: {} + disable_dualstack: false + gcs: + chunkBufferSize: 0 + requestTimeout: "0s" + enableHttp2: true + azure: + accountName: null + accountKey: null + connectionString: null + useManagedIdentity: false + useFederatedToken: false + userAssignedId: null + requestTimeout: null + endpointSuffix: null + chunkDelimiter: null + swift: + auth_version: null + auth_url: null + internal: null + username: null + user_domain_name: null + user_domain_id: null + user_id: null + password: null + domain_id: null + domain_name: null + project_id: null + project_name: null + project_domain_id: null + project_domain_name: null + region_name: null + container_name: null + max_retries: null + connect_timeout: null + request_timeout: null + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + admin_api_directory: /var/loki/admin + # -- Configure memcached as an external cache for chunk and results cache. Disabled by default + # must enable and specify a host for each cache you would like to use. + memcached: + chunk_cache: + enabled: false + host: "" + service: "memcached-client" + batch_size: 256 + parallelism: 10 + results_cache: + enabled: false + host: "" + service: "memcached-client" + timeout: "500ms" + default_validity: "12h" + # -- Check https://grafana.com/docs/loki/latest/configuration/#schema_config for more info on how to configure schemas + schemaConfig: {} + # -- a real Loki install requires a proper schemaConfig defined above this, however for testing or playing around + # you can enable useTestSchema + useTestSchema: false + testSchemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: '{{ include "loki.testSchemaObjectStore" . }}' + schema: v13 + index: + prefix: index_ + period: 24h + # -- Check https://grafana.com/docs/loki/latest/configuration/#ruler for more info on configuring ruler + rulerConfig: + wal: + dir: /var/loki/ruler-wal + # -- Structured loki configuration, takes precedence over `loki.config`, `loki.schemaConfig`, `loki.storageConfig` + structuredConfig: {} + # -- Additional query scheduler config + query_scheduler: {} + # -- Additional storage config + storage_config: + boltdb_shipper: + index_gateway_client: + server_address: '{{ include "loki.indexGatewayAddress" . }}' + tsdb_shipper: + index_gateway_client: + server_address: '{{ include "loki.indexGatewayAddress" . }}' + bloom_shipper: + working_directory: /var/loki/data/bloomshipper + hedging: + at: "250ms" + max_per_second: 20 + up_to: 3 + # -- Optional compactor configuration + compactor: {} + # -- Optional pattern ingester configuration + pattern_ingester: + enabled: false + # -- Optional analytics configuration + analytics: {} + # -- Optional querier configuration + query_range: {} + # -- Optional querier configuration + querier: {} + # -- Optional ingester configuration + ingester: {} + # -- Optional index gateway configuration + index_gateway: + mode: simple + frontend: + scheduler_address: '{{ include "loki.querySchedulerAddress" . }}' + tail_proxy_url: '{{ include "loki.querierAddress" . }}' + frontend_worker: + scheduler_address: '{{ include "loki.querySchedulerAddress" . }}' + # -- Optional distributor configuration + distributor: {} + # -- Enable tracing + tracing: + enabled: false + bloom_build: + enabled: false + builder: + planner_address: '{{ include "loki.bloomPlannerAddress" . }}' + bloom_gateway: + enabled: false + client: + addresses: '{{ include "loki.bloomGatewayAddresses" . }}' +###################################################################################################################### +# +# Enterprise Loki Configs +# +###################################################################################################################### + +# -- Configuration for running Enterprise Loki +enterprise: + # Enable enterprise features, license must be provided + enabled: false + # Default verion of GEL to deploy + version: 3.1.1 + # -- Optional name of the GEL cluster, otherwise will use .Release.Name + # The cluster name must match what is in your GEL license + cluster_name: null + # -- Grafana Enterprise Logs license + # In order to use Grafana Enterprise Logs features, you will need to provide + # the contents of your Grafana Enterprise Logs license, either by providing the + # contents of the license.jwt, or the name Kubernetes Secret that contains your + # license.jwt. + # To set the license contents, use the flag `--set-file 'enterprise.license.contents=./license.jwt'` + license: + contents: "NOTAVALIDLICENSE" + # -- Set to true when providing an external license + useExternalLicense: false + # -- Name of external license secret to use + externalLicenseName: null + # -- Name of the external config secret to use + externalConfigName: "" + # -- Use GEL gateway, if false will use the default nginx gateway + gelGateway: true + # -- If enabled, the correct admin_client storage will be configured. If disabled while running enterprise, + # make sure auth is set to `type: trust`, or that `auth_enabled` is set to `false`. + adminApi: + enabled: true + # enterprise specific sections of the config.yaml file + config: | + {{- if .Values.enterprise.adminApi.enabled }} + admin_client: + {{ include "enterprise-logs.adminAPIStorageConfig" . | nindent 2 }} + {{ end }} + auth: + type: {{ .Values.enterprise.adminApi.enabled | ternary "enterprise" "trust" }} + auth_enabled: {{ .Values.loki.auth_enabled }} + cluster_name: {{ include "loki.clusterName" . }} + license: + path: /etc/loki/license/license.jwt + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/enterprise-logs + # -- Docker image tag + tag: 3.3.1 + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent + adminToken: + # -- Alternative name for admin token secret, needed by tokengen and provisioner jobs + secret: null + # -- Additional namespace to also create the token in. Useful if your Grafana instance + # is in a different namespace + additionalNamespaces: [] + # -- Alternative name of the secret to store token for the canary + canarySecret: null + # -- Configuration for `tokengen` target + tokengen: + # -- Whether the job should be part of the deployment + enabled: true + # -- Comma-separated list of Loki modules to load for tokengen + targetModule: "tokengen" + # -- Additional CLI arguments for the `tokengen` target + extraArgs: [] + # -- Additional Kubernetes environment + env: [] + # -- Additional labels for the `tokengen` Job + labels: {} + # -- Additional annotations for the `tokengen` Job + annotations: {} + # -- Affinity for tokengen Pods + affinity: {} + # -- Node selector for tokengen Pods + nodeSelector: {} + # -- Tolerations for tokengen Job + tolerations: [] + # -- Additional volumes for Pods + extraVolumes: [] + # -- Additional volume mounts for Pods + extraVolumeMounts: [] + # -- Run containers as user `enterprise-logs(uid=10001)` + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + runAsUser: 10001 + fsGroup: 10001 + # -- Environment variables from secrets or configmaps to add to the tokengen pods + extraEnvFrom: [] + # -- The name of the PriorityClass for tokengen Pods + priorityClassName: "" + # -- Configuration for `provisioner` target + provisioner: + # -- Whether the job should be part of the deployment + enabled: true + # -- Name of the secret to store provisioned tokens in + provisionedSecretPrefix: null + # -- Additional tenants to be created. Each tenant will get a read and write policy + # and associated token. Tenant must have a name and a namespace for the secret containting + # the token to be created in. For example + # additionalTenants: + # - name: loki + # secretNamespace: grafana + additionalTenants: [] + # -- Additional Kubernetes environment + env: [] + # -- Additional labels for the `provisioner` Job + labels: {} + # -- Additional annotations for the `provisioner` Job + annotations: {} + # -- Affinity for tokengen Pods + affinity: {} + # -- Node selector for tokengen Pods + nodeSelector: {} + # -- Tolerations for tokengen Pods + tolerations: [] + # -- The name of the PriorityClass for provisioner Job + priorityClassName: null + # -- Run containers as user `enterprise-logs(uid=10001)` + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + runAsUser: 10001 + fsGroup: 10001 + # -- Provisioner image to Utilize + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/enterprise-logs-provisioner + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Volume mounts to add to the provisioner pods + extraVolumeMounts: [] +# -- kubetclImage is used in the enterprise provisioner and tokengen jobs +kubectlImage: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: bitnami/kubectl + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent +###################################################################################################################### +# +# Chart Testing +# +###################################################################################################################### + +# -- Section for configuring optional Helm test +test: + enabled: true + # -- Used to directly query the metrics endpoint of the canary for testing, this approach avoids needing prometheus for testing. + # This in a newer approach to using prometheusAddress such that tests do not have a dependency on prometheus + canaryServiceAddress: "http://loki-canary:3500/metrics" + # -- Address of the prometheus server to query for the test. This overrides any value set for canaryServiceAddress. + # This is kept for backward compatibility and may be removed in future releases. Previous value was 'http://prometheus:9090' + prometheusAddress: "" + # -- Number of times to retry the test before failing + timeout: 1m + # -- Additional labels for the test pods + labels: {} + # -- Additional annotations for test pods + annotations: {} + # -- Image to use for loki canary + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/loki-helm-test + # -- Overrides the image tag whose default is the chart's appVersion + tag: "ewelch-distributed-helm-chart-17db5ee" + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent +# The Loki canary pushes logs to and queries from this loki installation to test +# that it's working correctly +lokiCanary: + enabled: true + # -- If true, the canary will send directly to Loki via the address configured for verification -- + # -- If false, it will write to stdout and an Agent will be needed to scrape and send the logs -- + push: true + # -- The name of the label to look for at loki when doing the checks. + labelname: pod + # -- Additional annotations for the `loki-canary` Daemonset + annotations: {} + # -- Additional labels for each `loki-canary` pod + podLabels: {} + service: + # -- Annotations for loki-canary Service + annotations: {} + # -- Additional labels for loki-canary Service + labels: {} + # -- Additional CLI arguments for the `loki-canary' command + extraArgs: [] + # -- Environment variables to add to the canary pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the canary pods + extraEnvFrom: [] + # -- Volume mounts to add to the canary pods + extraVolumeMounts: [] + # -- Volumes to add to the canary pods + extraVolumes: [] + # -- Resource requests and limits for the canary + resources: {} + # -- DNS config for canary pods + dnsConfig: {} + # -- Node selector for canary pods + nodeSelector: {} + # -- Tolerations for canary pods + tolerations: [] + # -- The name of the PriorityClass for loki-canary pods + priorityClassName: null + # -- Image to use for loki canary + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/loki-canary + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + # -- Overrides the image tag with an image digest + digest: null + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Update strategy for the `loki-canary` Daemonset pods + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 +###################################################################################################################### +# +# Service Accounts and Kubernetes RBAC +# +###################################################################################################################### +serviceAccount: + # -- Specifies whether a ServiceAccount should be created + create: true + # -- The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: null + # -- Image pull secrets for the service account + imagePullSecrets: [] + # -- Annotations for the service account + annotations: {} + # -- Labels for the service account + labels: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true +# RBAC configuration +rbac: + # -- If pspEnabled true, a PodSecurityPolicy is created for K8s that use psp. + pspEnabled: false + # -- For OpenShift set pspEnabled to 'false' and sccEnabled to 'true' to use the SecurityContextConstraints. + sccEnabled: false + # -- Specify PSP annotations + # Ref: https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/#podsecuritypolicy-annotations + pspAnnotations: {} + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + # -- Whether to install RBAC in the namespace only or cluster-wide. Useful if you want to watch ConfigMap globally. + namespaced: false +###################################################################################################################### +# +# Network Policy configuration +# +###################################################################################################################### +networkPolicy: + # -- Specifies whether Network Policies should be created + enabled: false + # -- Specifies whether the policies created will be standard Network Policies (flavor: kubernetes) + # or Cilium Network Policies (flavor: cilium) + flavor: kubernetes + metrics: + # -- Specifies the Pods which are allowed to access the metrics port. + # As this is cross-namespace communication, you also need the namespaceSelector. + podSelector: {} + # -- Specifies the namespaces which are allowed to access the metrics port + namespaceSelector: {} + # -- Specifies specific network CIDRs which are allowed to access the metrics port. + # In case you use namespaceSelector, you also have to specify your kubelet networks here. + # The metrics ports are also used for probes. + cidrs: [] + ingress: + # -- Specifies the Pods which are allowed to access the http port. + # As this is cross-namespace communication, you also need the namespaceSelector. + podSelector: {} + # -- Specifies the namespaces which are allowed to access the http port + namespaceSelector: {} + alertmanager: + # -- Specify the alertmanager port used for alerting + port: 9093 + # -- Specifies the alertmanager Pods. + # As this is cross-namespace communication, you also need the namespaceSelector. + podSelector: {} + # -- Specifies the namespace the alertmanager is running in + namespaceSelector: {} + externalStorage: + # -- Specify the port used for external storage, e.g. AWS S3 + ports: [] + # -- Specifies specific network CIDRs you want to limit access to + cidrs: [] + discovery: + # -- (int) Specify the port used for discovery + port: null + # -- Specifies the Pods labels used for discovery. + # As this is cross-namespace communication, you also need the namespaceSelector. + podSelector: {} + # -- Specifies the namespace the discovery Pods are running in + namespaceSelector: {} + egressWorld: + # -- Enable additional cilium egress rules to external world for write, read and backend. + enabled: false + egressKubeApiserver: + # -- Enable additional cilium egress rules to kube-apiserver for backend. + enabled: false +###################################################################################################################### +# +# Global memberlist configuration +# +###################################################################################################################### + +# Configuration for the memberlist service +memberlist: + service: + publishNotReadyAddresses: false + annotations: {} +###################################################################################################################### +# +# adminAPI configuration, enterprise only. +# +###################################################################################################################### + +# -- Configuration for the `admin-api` target +adminApi: + # -- Define the amount of instances + replicas: 1 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + # -- Additional CLI arguments for the `admin-api` target + extraArgs: {} + # -- Environment variables from secrets or configmaps to add to the admin-api pods + extraEnvFrom: [] + # -- Additional labels for the `admin-api` Deployment + labels: {} + # -- Additional annotations for the `admin-api` Deployment + annotations: {} + # -- Additional labels and annotations for the `admin-api` Service + service: + labels: {} + annotations: {} + # -- Run container as user `enterprise-logs(uid=10001)` + # `fsGroup` must not be specified, because these security options are applied + # on container level not on Pod level. + podSecurityContext: + runAsNonRoot: true + runAsGroup: 10001 + runAsUser: 10001 + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + # -- Update strategy + strategy: + type: RollingUpdate + # -- Readiness probe + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + # -- Request and limit Kubernetes resources + # -- Values are defined in small.yaml and large.yaml + resources: {} + # -- Configure optional environment variables + env: [] + # -- Configure optional initContainers + initContainers: [] + # -- Conifgure optional extraContainers + extraContainers: [] + # -- Additional volumes for Pods + extraVolumes: [] + # -- Additional volume mounts for Pods + extraVolumeMounts: [] + # -- Affinity for admin-api Pods + affinity: {} + # -- Node selector for admin-api Pods + nodeSelector: {} + # -- Topology Spread Constraints for admin-api pods + topologySpreadConstraints: [] + # -- Tolerations for admin-api Pods + tolerations: [] + # -- Grace period to allow the admin-api to shutdown before it is killed + terminationGracePeriodSeconds: 60 +###################################################################################################################### +# +# Gateway and Ingress +# +# By default this chart will deploy a Nginx container to act as a gateway which handles routing of traffic +# and can also do auth. +# +# If you would prefer you can optionally disable this and enable using k8s ingress to do the incoming routing. +# +###################################################################################################################### + +# Configuration for the gateway +gateway: + # -- Specifies whether the gateway should be enabled + enabled: true + # -- Number of replicas for the gateway + replicas: 1 + # -- Default container port + containerPort: 8080 + # -- Enable logging of 2xx and 3xx HTTP requests + verboseLogging: true + autoscaling: + # -- Enable autoscaling for the gateway + enabled: false + # -- Minimum autoscaling replicas for the gateway + minReplicas: 1 + # -- Maximum autoscaling replicas for the gateway + maxReplicas: 3 + # -- Target CPU utilisation percentage for the gateway + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the gateway + targetMemoryUtilizationPercentage: + # -- See `kubectl explain deployment.spec.strategy` for more + # -- ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + # -- Behavior policies while scaling. + behavior: {} + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 60 + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + deploymentStrategy: + type: RollingUpdate + image: + # -- The Docker registry for the gateway image + registry: docker.io + # -- The gateway image repository + repository: nginxinc/nginx-unprivileged + # -- The gateway image tag + tag: 1.27-alpine + # -- Overrides the gateway image tag with an image digest + digest: null + # -- The gateway image pull policy + pullPolicy: IfNotPresent + # -- The name of the PriorityClass for gateway pods + priorityClassName: null + # -- Annotations for gateway deployment + annotations: {} + # -- Annotations for gateway pods + podAnnotations: {} + # -- Additional labels for gateway pods + podLabels: {} + # -- Additional CLI args for the gateway + extraArgs: [] + # -- Environment variables to add to the gateway pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the gateway pods + extraEnvFrom: [] + # -- Lifecycle for the gateway container + lifecycle: {} + # -- Volumes to add to the gateway pods + extraVolumes: [] + # -- Volume mounts to add to the gateway pods + extraVolumeMounts: [] + # -- The SecurityContext for gateway containers + podSecurityContext: + fsGroup: 101 + runAsGroup: 101 + runAsNonRoot: true + runAsUser: 101 + # -- The SecurityContext for gateway containers + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + # -- Resource requests and limits for the gateway + resources: {} + # -- Containers to add to the gateway pods + extraContainers: [] + # -- Grace period to allow the gateway to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for gateway pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: gateway + topologyKey: kubernetes.io/hostname + # -- DNS config for gateway pods + dnsConfig: {} + # -- Node selector for gateway pods + nodeSelector: {} + # -- Topology Spread Constraints for gateway pods + topologySpreadConstraints: [] + # -- Tolerations for gateway pods + tolerations: [] + # Gateway service configuration + service: + # -- Port of the gateway service + port: 80 + # -- Type of the gateway service + type: ClusterIP + # -- ClusterIP of the gateway service + clusterIP: null + # -- (int) Node port if service type is NodePort + nodePort: null + # -- Load balancer IPO address if service type is LoadBalancer + loadBalancerIP: null + # -- Annotations for the gateway service + annotations: {} + # -- Labels for gateway service + labels: {} + # Gateway ingress configuration + ingress: + # -- Specifies whether an ingress for the gateway should be created + enabled: false + # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 + ingressClassName: "" + # -- Annotations for the gateway ingress + annotations: {} + # -- Labels for the gateway ingress + labels: {} + # -- Hosts configuration for the gateway ingress, passed through the `tpl` function to allow templating + hosts: + - host: gateway.loki.example.com + paths: + - path: / + # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers + # pathType: Prefix + # -- TLS configuration for the gateway ingress. Hosts passed through the `tpl` function to allow templating + tls: + - secretName: loki-gateway-tls + hosts: + - gateway.loki.example.com + # Basic auth configuration + basicAuth: + # -- Enables basic authentication for the gateway + enabled: false + # -- The basic auth username for the gateway + username: null + # -- The basic auth password for the gateway + password: null + # -- Uses the specified users from the `loki.tenants` list to create the htpasswd file. + # if `loki.tenants` is not set, the `gateway.basicAuth.username` and `gateway.basicAuth.password` are used. + # The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes + # high CPU load. + # @default -- Either `loki.tenants` or `gateway.basicAuth.username` and `gateway.basicAuth.password`. + htpasswd: >- + {{ if .Values.loki.tenants }} + + + {{- range $t := .Values.loki.tenants }} + {{ htpasswd (required "All tenants must have a 'name' set" $t.name) (required "All tenants must have a 'password' set" $t.password) }} + + + {{- end }} + {{ else }} {{ htpasswd (required "'gateway.basicAuth.username' is required" .Values.gateway.basicAuth.username) (required "'gateway.basicAuth.password' is required" .Values.gateway.basicAuth.password) }} {{ end }} + # -- Existing basic auth secret to use. Must contain '.htpasswd' + existingSecret: null + # Configures the readiness probe for the gateway + readinessProbe: + httpGet: + path: / + port: http-metrics + initialDelaySeconds: 15 + timeoutSeconds: 1 + nginxConfig: + # -- Which schema to be used when building URLs. Can be 'http' or 'https'. + schema: http + # -- Enable listener for IPv6, disable on IPv4-only systems + enableIPv6: true + # -- NGINX log format + logFormat: |- + main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + # -- Allows appending custom configuration to the server block + serverSnippet: "" + # -- Allows appending custom configuration to the http block, passed through the `tpl` function to allow templating + httpSnippet: >- + {{ if .Values.loki.tenants }}proxy_set_header X-Scope-OrgID $remote_user;{{ end }} + # -- Allows customizing the `client_max_body_size` directive + clientMaxBodySize: 4M + # -- Whether ssl should be appended to the listen directive of the server block or not. + ssl: false + # -- Override Read URL + customReadUrl: null + # -- Override Write URL + customWriteUrl: null + # -- Override Backend URL + customBackendUrl: null + # -- Allows overriding the DNS resolver address nginx will use. + resolver: "" + # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating + # @default -- See values.yaml + file: | + {{- include "loki.nginxFile" . | indent 2 -}} +# -- If running enterprise and using the default enterprise gateway, configs go here. +enterpriseGateway: + # -- Define the amount of instances + replicas: 1 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + # -- Additional CLI arguments for the `gateway` target + extraArgs: {} + # -- Environment variables from secrets or configmaps to add to the enterprise gateway pods + extraEnvFrom: [] + # -- Additional labels for the `gateway` Pod + labels: {} + # -- Additional annotations for the `gateway` Pod + annotations: {} + # -- Additional labels and annotations for the `gateway` Service + # -- Service overriding service type + service: + type: ClusterIP + labels: {} + annotations: {} + # -- Run container as user `enterprise-logs(uid=10001)` + podSecurityContext: + runAsNonRoot: true + runAsGroup: 10001 + runAsUser: 10001 + fsGroup: 10001 + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + # -- If you want to use your own proxy URLs, set this to false. + useDefaultProxyURLs: true + # -- update strategy + strategy: + type: RollingUpdate + # -- Readiness probe + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + # -- Request and limit Kubernetes resources + # -- Values are defined in small.yaml and large.yaml + resources: {} + # -- Configure optional environment variables + env: [] + # -- Configure optional initContainers + initContainers: [] + # -- Conifgure optional extraContainers + extraContainers: [] + # -- Additional volumes for Pods + extraVolumes: [] + # -- Additional volume mounts for Pods + extraVolumeMounts: [] + # -- Affinity for gateway Pods + affinity: {} + # -- Node selector for gateway Pods + nodeSelector: {} + # -- Topology Spread Constraints for enterprise-gateway pods + topologySpreadConstraints: [] + # -- Tolerations for gateway Pods + tolerations: [] + # -- Grace period to allow the gateway to shutdown before it is killed + terminationGracePeriodSeconds: 60 +# -- Ingress configuration Use either this ingress or the gateway, but not both at once. +# If you enable this, make sure to disable the gateway. +# You'll need to supply authn configuration for your ingress controller. +ingress: + enabled: false + ingressClassName: "" + annotations: {} + # nginx.ingress.kubernetes.io/auth-type: basic + # nginx.ingress.kubernetes.io/auth-secret: loki-distributed-basic-auth + # nginx.ingress.kubernetes.io/auth-secret-type: auth-map + # nginx.ingress.kubernetes.io/configuration-snippet: | + # proxy_set_header X-Scope-OrgID $remote_user; + labels: {} + # blackbox.monitoring.exclude: "true" + paths: + # -- Paths that are exposed by Loki Distributor. + # If deployment mode is Distributed, the requests are forwarded to the service: `{{"loki.distributorFullname"}}`. + # If deployment mode is SimpleScalable, the requests are forwarded to write k8s service: `{{"loki.writeFullname"}}`. + # If deployment mode is SingleBinary, the requests are forwarded to the central/single k8s service: `{{"loki.singleBinaryFullname"}}` + distributor: + - /api/prom/push + - /loki/api/v1/push + - /otlp/v1/logs + # -- Paths that are exposed by Loki Query Frontend. + # If deployment mode is Distributed, the requests are forwarded to the service: `{{"loki.queryFrontendFullname"}}`. + # If deployment mode is SimpleScalable, the requests are forwarded to write k8s service: `{{"loki.readFullname"}}`. + # If deployment mode is SingleBinary, the requests are forwarded to the central/single k8s service: `{{"loki.singleBinaryFullname"}}` + queryFrontend: + - /api/prom/query + # this path covers labels and labelValues endpoints + - /api/prom/label + - /api/prom/series + - /api/prom/tail + - /loki/api/v1/query + - /loki/api/v1/query_range + - /loki/api/v1/tail + # this path covers labels and labelValues endpoints + - /loki/api/v1/label + - /loki/api/v1/labels + - /loki/api/v1/series + - /loki/api/v1/index/stats + - /loki/api/v1/index/volume + - /loki/api/v1/index/volume_range + - /loki/api/v1/format_query + - /loki/api/v1/detected_field + - /loki/api/v1/detected_fields + - /loki/api/v1/detected_labels + - /loki/api/v1/patterns + # -- Paths that are exposed by Loki Ruler. + # If deployment mode is Distributed, the requests are forwarded to the service: `{{"loki.rulerFullname"}}`. + # If deployment mode is SimpleScalable, the requests are forwarded to k8s service: `{{"loki.backendFullname"}}`. + # If deployment mode is SimpleScalable but `read.legacyReadTarget` is `true`, the requests are forwarded to k8s service: `{{"loki.readFullname"}}`. + # If deployment mode is SingleBinary, the requests are forwarded to the central/single k8s service: `{{"loki.singleBinaryFullname"}}` + ruler: + - /api/prom/rules + - /api/prom/api/v1/rules + - /api/prom/api/v1/alerts + - /loki/api/v1/rules + - /prometheus/api/v1/rules + - /prometheus/api/v1/alerts + # -- Hosts configuration for the ingress, passed through the `tpl` function to allow templating + hosts: + - loki.example.com + # -- TLS configuration for the ingress. Hosts passed through the `tpl` function to allow templating + tls: [] +# - hosts: +# - loki.example.com +# secretName: loki-distributed-tls + +###################################################################################################################### +# +# Migration +# +###################################################################################################################### + +# -- Options that may be necessary when performing a migration from another helm chart +migrate: + # -- When migrating from a distributed chart like loki-distributed or enterprise-logs + fromDistributed: + # -- Set to true if migrating from a distributed helm chart + enabled: false + # -- If migrating from a distributed service, provide the distributed deployment's + # memberlist service DNS so the new deployment can join its ring. + memberlistService: "" +###################################################################################################################### +# +# Single Binary Deployment +# +# For small Loki installations up to a few 10's of GB per day, or for testing and development. +# +###################################################################################################################### + +# Configuration for the single binary node(s) +singleBinary: + # -- Number of replicas for the single binary + replicas: 0 + autoscaling: + # -- Enable autoscaling + enabled: false + # -- Minimum autoscaling replicas for the single binary + minReplicas: 1 + # -- Maximum autoscaling replicas for the single binary + maxReplicas: 3 + # -- Target CPU utilisation percentage for the single binary + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the single binary + targetMemoryUtilizationPercentage: + image: + # -- The Docker registry for the single binary image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the single binary image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the single binary image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for single binary pods + priorityClassName: null + # -- Annotations for single binary StatefulSet + annotations: {} + # -- Annotations for single binary pods + podAnnotations: {} + # -- Additional labels for each `single binary` pod + podLabels: {} + # -- Additional selector labels for each `single binary` pod + selectorLabels: {} + service: + # -- Annotations for single binary Service + annotations: {} + # -- Additional labels for single binary Service + labels: {} + # -- Comma-separated list of Loki modules to load for the single binary + targetModule: "all" + # -- Labels for single binary service + extraArgs: [] + # -- Environment variables to add to the single binary pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the single binary pods + extraEnvFrom: [] + # -- Extra containers to add to the single binary loki pod + extraContainers: [] + # -- Init containers to add to the single binary pods + initContainers: [] + # -- Volume mounts to add to the single binary pods + extraVolumeMounts: [] + # -- Volumes to add to the single binary pods + extraVolumes: [] + # -- Resource requests and limits for the single binary + resources: {} + # -- Grace period to allow the single binary to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for single binary pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: single-binary + topologyKey: kubernetes.io/hostname + # -- DNS config for single binary pods + dnsConfig: {} + # -- Node selector for single binary pods + nodeSelector: {} + # -- Tolerations for single binary pods + tolerations: [] + persistence: + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: true + # -- Enable persistent disk + enabled: true + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Selector for persistent disk + selector: null + # -- Annotations for volume claim + annotations: {} +###################################################################################################################### +# +# Simple Scalable Deployment (SSD) Mode +# +# For small to medium size Loki deployments up to around 1 TB/day, this is the default mode for this helm chart +# +###################################################################################################################### + +# Configuration for the write pod(s) +write: + # -- Number of replicas for the write + replicas: 3 + autoscaling: + # -- Enable autoscaling for the write. + enabled: false + # -- Minimum autoscaling replicas for the write. + minReplicas: 2 + # -- Maximum autoscaling replicas for the write. + maxReplicas: 6 + # -- Target CPU utilisation percentage for the write. + targetCPUUtilizationPercentage: 60 + # -- Target memory utilization percentage for the write. + targetMemoryUtilizationPercentage: + # -- Behavior policies while scaling. + behavior: + # -- see https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/wal.md#how-to-scale-updown for scaledown details + scaleUp: + policies: + - type: Pods + value: 1 + periodSeconds: 900 + scaleDown: + policies: + - type: Pods + value: 1 + periodSeconds: 1800 + stabilizationWindowSeconds: 3600 + image: + # -- The Docker registry for the write image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the write image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the write image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for write pods + priorityClassName: null + # -- Annotations for write StatefulSet + annotations: {} + # -- Annotations for write pods + podAnnotations: {} + # -- Additional labels for each `write` pod + podLabels: {} + # -- Additional selector labels for each `write` pod + selectorLabels: {} + service: + # -- Annotations for write Service + annotations: {} + # -- Additional labels for write Service + labels: {} + # -- Comma-separated list of Loki modules to load for the write + targetModule: "write" + # -- Additional CLI args for the write + extraArgs: [] + # -- Environment variables to add to the write pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the write pods + extraEnvFrom: [] + # -- Lifecycle for the write container + lifecycle: {} + # -- The default /flush_shutdown preStop hook is recommended as part of the ingester + # scaledown process so it's added to the template by default when autoscaling is enabled, + # but it's disabled to optimize rolling restarts in instances that will never be scaled + # down or when using chunks storage with WAL disabled. + # https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/wal.md#how-to-scale-updown + # -- Init containers to add to the write pods + initContainers: [] + # -- Containers to add to the write pods + extraContainers: [] + # -- Volume mounts to add to the write pods + extraVolumeMounts: [] + # -- Volumes to add to the write pods + extraVolumes: [] + # -- volumeClaimTemplates to add to StatefulSet + extraVolumeClaimTemplates: [] + # -- Resource requests and limits for the write + resources: {} + # -- Grace period to allow the write to shutdown before it is killed. Especially for the ingester, + # this must be increased. It must be long enough so writes can be gracefully shutdown flushing/transferring + # all data and to successfully leave the member ring on shutdown. + terminationGracePeriodSeconds: 300 + # -- Affinity for write pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: write + topologyKey: kubernetes.io/hostname + # -- DNS config for write pods + dnsConfig: {} + # -- Node selector for write pods + nodeSelector: {} + # -- Topology Spread Constraints for write pods + topologySpreadConstraints: [] + # -- Tolerations for write pods + tolerations: [] + # -- The default is to deploy all pods in parallel. + podManagementPolicy: "Parallel" + persistence: + # -- Enable volume claims in pod spec + volumeClaimsEnabled: true + # -- Parameters used for the `data` volume when volumeClaimEnabled if false + dataVolumeParameters: + emptyDir: {} + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Selector for persistent disk + selector: null + # -- Annotations for volume claim + annotations: {} +# -- Configuration for the read pod(s) +read: + # -- Number of replicas for the read + replicas: 3 + autoscaling: + # -- Enable autoscaling for the read, this is only used if `queryIndex.enabled: true` + enabled: false + # -- Minimum autoscaling replicas for the read + minReplicas: 2 + # -- Maximum autoscaling replicas for the read + maxReplicas: 6 + # -- Target CPU utilisation percentage for the read + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the read + targetMemoryUtilizationPercentage: + # -- Behavior policies while scaling. + behavior: {} + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 60 + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + image: + # -- The Docker registry for the read image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the read image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the read image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for read pods + priorityClassName: null + # -- Annotations for read deployment + annotations: {} + # -- Annotations for read pods + podAnnotations: {} + # -- Additional labels for each `read` pod + podLabels: {} + # -- Additional selector labels for each `read` pod + selectorLabels: {} + service: + # -- Annotations for read Service + annotations: {} + # -- Additional labels for read Service + labels: {} + # -- Comma-separated list of Loki modules to load for the read + targetModule: "read" + # -- Whether or not to use the 2 target type simple scalable mode (read, write) or the + # 3 target type (read, write, backend). Legacy refers to the 2 target type, so true will + # run two targets, false will run 3 targets. + legacyReadTarget: false + # -- Additional CLI args for the read + extraArgs: [] + # -- Containers to add to the read pods + extraContainers: [] + # -- Environment variables to add to the read pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the read pods + extraEnvFrom: [] + # -- Lifecycle for the read container + lifecycle: {} + # -- Volume mounts to add to the read pods + extraVolumeMounts: [] + # -- Volumes to add to the read pods + extraVolumes: [] + # -- Resource requests and limits for the read + resources: {} + # -- Grace period to allow the read to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for read pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: read + topologyKey: kubernetes.io/hostname + # -- DNS config for read pods + dnsConfig: {} + # -- Node selector for read pods + nodeSelector: {} + # -- Topology Spread Constraints for read pods + topologySpreadConstraints: [] + # -- Tolerations for read pods + tolerations: [] + # -- The default is to deploy all pods in parallel. + podManagementPolicy: "Parallel" + persistence: + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: true + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Selector for persistent disk + selector: null + # -- Annotations for volume claim + annotations: {} +# -- Configuration for the backend pod(s) +backend: + # -- Number of replicas for the backend + replicas: 3 + autoscaling: + # -- Enable autoscaling for the backend. + enabled: false + # -- Minimum autoscaling replicas for the backend. + minReplicas: 3 + # -- Maximum autoscaling replicas for the backend. + maxReplicas: 6 + # -- Target CPU utilization percentage for the backend. + targetCPUUtilizationPercentage: 60 + # -- Target memory utilization percentage for the backend. + targetMemoryUtilizationPercentage: + # -- Behavior policies while scaling. + behavior: {} + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 60 + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + image: + # -- The Docker registry for the backend image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the backend image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the backend image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for backend pods + priorityClassName: null + # -- Annotations for backend StatefulSet + annotations: {} + # -- Annotations for backend pods + podAnnotations: {} + # -- Additional labels for each `backend` pod + podLabels: {} + # -- Additional selector labels for each `backend` pod + selectorLabels: {} + service: + # -- Annotations for backend Service + annotations: {} + # -- Additional labels for backend Service + labels: {} + # -- Comma-separated list of Loki modules to load for the backend + targetModule: "backend" + # -- Additional CLI args for the backend + extraArgs: [] + # -- Environment variables to add to the backend pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the backend pods + extraEnvFrom: [] + # -- Init containers to add to the backend pods + initContainers: [] + # -- Volume mounts to add to the backend pods + extraVolumeMounts: [] + # -- Volumes to add to the backend pods + extraVolumes: [] + # -- Resource requests and limits for the backend + resources: {} + # -- Grace period to allow the backend to shutdown before it is killed. Especially for the ingester, + # this must be increased. It must be long enough so backends can be gracefully shutdown flushing/transferring + # all data and to successfully leave the member ring on shutdown. + terminationGracePeriodSeconds: 300 + # -- Affinity for backend pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: backend + topologyKey: kubernetes.io/hostname + # -- DNS config for backend pods + dnsConfig: {} + # -- Node selector for backend pods + nodeSelector: {} + # -- Topology Spread Constraints for backend pods + topologySpreadConstraints: [] + # -- Tolerations for backend pods + tolerations: [] + # -- The default is to deploy all pods in parallel. + podManagementPolicy: "Parallel" + persistence: + # -- Enable volume claims in pod spec + volumeClaimsEnabled: true + # -- Parameters used for the `data` volume when volumeClaimEnabled if false + dataVolumeParameters: + emptyDir: {} + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: true + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Selector for persistent disk + selector: null + # -- Annotations for volume claim + annotations: {} +###################################################################################################################### +# +# Microservices Mode +# +# For large Loki deployments ingesting more than 1 TB/day +# +###################################################################################################################### + +# -- Configuration for the ingester +ingester: + # -- Number of replicas for the ingester, when zoneAwareReplication.enabled is true, the total + # number of replicas will match this value with each zone having 1/3rd of the total replicas. + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + autoscaling: + # -- Enable autoscaling for the ingester + enabled: false + # -- Minimum autoscaling replicas for the ingester + minReplicas: 1 + # -- Maximum autoscaling replicas for the ingester + maxReplicas: 3 + # -- Target CPU utilisation percentage for the ingester + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the ingester + targetMemoryUtilizationPercentage: null + # -- Allows one to define custom metrics using the HPA/v2 schema (for example, Pods, Object or External metrics) + customMetrics: [] + # - type: Pods + # pods: + # metric: + # name: loki_lines_total + # target: + # type: AverageValue + # averageValue: 10k + behavior: + # -- Enable autoscaling behaviours + enabled: false + # -- define scale down policies, must conform to HPAScalingRules + scaleDown: {} + # -- define scale up policies, must conform to HPAScalingRules + scaleUp: {} + image: + # -- The Docker registry for the ingester image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the ingester image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the ingester image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + priorityClassName: null + # -- Labels for ingester pods + podLabels: {} + # -- Annotations for ingester pods + podAnnotations: {} + # -- The name of the PriorityClass for ingester pods + # -- Labels for ingestor service + serviceLabels: {} + # -- Annotations for ingestor service + serviceAnnotations: {} + # -- Additional CLI args for the ingester + extraArgs: [] + # -- Environment variables to add to the ingester pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the ingester pods + extraEnvFrom: [] + # -- Volume mounts to add to the ingester pods + extraVolumeMounts: [] + # -- Volumes to add to the ingester pods + extraVolumes: [] + # -- Resource requests and limits for the ingester + resources: {} + # -- Containers to add to the ingester pods + extraContainers: [] + # -- Init containers to add to the ingester pods + initContainers: [] + # -- Grace period to allow the ingester to shutdown before it is killed. Especially for the ingestor, + # this must be increased. It must be long enough so ingesters can be gracefully shutdown flushing/transferring + # all data and to successfully leave the member ring on shutdown. + terminationGracePeriodSeconds: 300 + # -- Lifecycle for the ingester container + lifecycle: {} + # -- topologySpread for ingester pods. + # @default -- Defaults to allow skew no more than 1 node + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: ingester + # -- Affinity for ingester pods. Ignored if zoneAwareReplication is enabled. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: ingester + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: 1 + # -- Node selector for ingester pods + nodeSelector: {} + # -- Tolerations for ingester pods + tolerations: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- UpdateStrategy for the ingester StatefulSets. + updateStrategy: + # -- One of 'OnDelete' or 'RollingUpdate' + type: RollingUpdate + # -- Optional for updateStrategy.type=RollingUpdate. See [Partitioned rolling updates](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) in the StatefulSet docs for details. + # rollingUpdate: + # partition: 0 + persistence: + # -- Enable creating PVCs which is required when using boltdb-shipper + enabled: false + # -- Use emptyDir with ramdisk for storage. **Please note that all data in ingester will be lost on pod restart** + inMemory: false + # -- List of the ingester PVCs + # @notationType -- list + claims: + - name: data + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # - name: wal + # size: 150Gi + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + # -- Adds the appProtocol field to the ingester service. This allows ingester to work with istio protocol selection. + appProtocol: + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + grpc: "" + # -- Enabling zone awareness on ingesters will create 3 statefulests where all writes will send a replica to each zone. + # This is primarily intended to accelerate rollout operations by allowing for multiple ingesters within a single + # zone to be shutdown and restart simultaneously (the remaining 2 zones will be guaranteed to have at least one copy + # of the data). + # Note: This can be used to run Loki over multiple cloud provider availability zones however this is not currently + # recommended as Loki is not optimized for this and cross zone network traffic costs can become extremely high + # extremely quickly. Even with zone awareness enabled, it is recommended to run Loki in a single availability zone. + zoneAwareReplication: + # -- Enable zone awareness. + enabled: true + # -- The percent of replicas in each zone that will be restarted at once. In a value of 0-100 + maxUnavailablePct: 33 + # -- zoneA configuration + zoneA: + # -- optionally define a node selector for this zone + nodeSelector: null + # -- optionally define extra affinity rules, by default different zones are not allowed to schedule on the same host + extraAffinity: {} + # -- Specific annotations to add to zone A statefulset + annotations: {} + # -- Specific annotations to add to zone A pods + podAnnotations: {} + zoneB: + # -- optionally define a node selector for this zone + nodeSelector: null + # -- optionally define extra affinity rules, by default different zones are not allowed to schedule on the same host + extraAffinity: {} + # -- Specific annotations to add to zone B statefulset + annotations: {} + # -- Specific annotations to add to zone B pods + podAnnotations: {} + zoneC: + # -- optionally define a node selector for this zone + nodeSelector: null + # -- optionally define extra affinity rules, by default different zones are not allowed to schedule on the same host + extraAffinity: {} + # -- Specific annotations to add to zone C statefulset + annotations: {} + # -- Specific annotations to add to zone C pods + podAnnotations: {} + # -- The migration block allows migrating non zone aware ingesters to zone aware ingesters. + migration: + enabled: false + excludeDefaultZone: false + readPath: false + writePath: false +# -- Configuration for the distributor +distributor: + # -- Number of replicas for the distributor + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + autoscaling: + # -- Enable autoscaling for the distributor + enabled: false + # -- Minimum autoscaling replicas for the distributor + minReplicas: 1 + # -- Maximum autoscaling replicas for the distributor + maxReplicas: 3 + # -- Target CPU utilisation percentage for the distributor + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the distributor + targetMemoryUtilizationPercentage: null + # -- Allows one to define custom metrics using the HPA/v2 schema (for example, Pods, Object or External metrics) + customMetrics: [] + # - type: Pods + # pods: + # metric: + # name: loki_lines_total + # target: + # type: AverageValue + # averageValue: 10k + behavior: + # -- Enable autoscaling behaviours + enabled: false + # -- define scale down policies, must conform to HPAScalingRules + scaleDown: {} + # -- define scale up policies, must conform to HPAScalingRules + scaleUp: {} + image: + # -- The Docker registry for the distributor image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the distributor image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the distributor image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for distributor pods + priorityClassName: null + # -- Labels for distributor pods + podLabels: {} + # -- Annotations for distributor pods + podAnnotations: {} + # -- Labels for distributor service + serviceLabels: {} + # -- Annotations for distributor service + serviceAnnotations: {} + # -- Additional CLI args for the distributor + extraArgs: [] + # -- Environment variables to add to the distributor pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the distributor pods + extraEnvFrom: [] + # -- Volume mounts to add to the distributor pods + extraVolumeMounts: [] + # -- Volumes to add to the distributor pods + extraVolumes: [] + # -- Resource requests and limits for the distributor + resources: {} + # -- Containers to add to the distributor pods + extraContainers: [] + # -- Grace period to allow the distributor to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for distributor pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: distributor + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Max Surge for distributor pods + maxSurge: 0 + # -- Node selector for distributor pods + nodeSelector: {} + # -- Topology Spread Constraints for distributor pods + topologySpreadConstraints: [] + # -- Tolerations for distributor pods + tolerations: [] + # -- Adds the appProtocol field to the distributor service. This allows distributor to work with istio protocol selection. + appProtocol: + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + grpc: "" +# -- Configuration for the querier +querier: + # -- Number of replicas for the querier + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + autoscaling: + # -- Enable autoscaling for the querier, this is only used if `indexGateway.enabled: true` + enabled: false + # -- Minimum autoscaling replicas for the querier + minReplicas: 1 + # -- Maximum autoscaling replicas for the querier + maxReplicas: 3 + # -- Target CPU utilisation percentage for the querier + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the querier + targetMemoryUtilizationPercentage: null + # -- Allows one to define custom metrics using the HPA/v2 schema (for example, Pods, Object or External metrics) + customMetrics: [] + # - type: External + # external: + # metric: + # name: loki_inflight_queries + # target: + # type: AverageValue + # averageValue: 12 + behavior: + # -- Enable autoscaling behaviours + enabled: false + # -- define scale down policies, must conform to HPAScalingRules + scaleDown: {} + # -- define scale up policies, must conform to HPAScalingRules + scaleUp: {} + image: + # -- The Docker registry for the querier image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the querier image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the querier image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for querier pods + priorityClassName: null + # -- Labels for querier pods + podLabels: {} + # -- Annotations for querier pods + podAnnotations: {} + # -- Labels for querier service + serviceLabels: {} + # -- Annotations for querier service + serviceAnnotations: {} + # -- Additional CLI args for the querier + extraArgs: [] + # -- Environment variables to add to the querier pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the querier pods + extraEnvFrom: [] + # -- Volume mounts to add to the querier pods + extraVolumeMounts: [] + # -- Volumes to add to the querier pods + extraVolumes: [] + # -- Resource requests and limits for the querier + resources: {} + # -- Containers to add to the querier pods + extraContainers: [] + # -- Init containers to add to the querier pods + initContainers: [] + # -- Grace period to allow the querier to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- topologySpread for querier pods. + # @default -- Defaults to allow skew no more then 1 node + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: querier + # -- Affinity for querier pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: querier + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Max Surge for querier pods + maxSurge: 0 + # -- Node selector for querier pods + nodeSelector: {} + # -- Tolerations for querier pods + tolerations: [] + # -- DNSConfig for querier pods + dnsConfig: {} + persistence: + # -- Enable creating PVCs for the querier cache + enabled: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for querier PVCs + annotations: {} + # -- Adds the appProtocol field to the querier service. This allows querier to work with istio protocol selection. + appProtocol: + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + grpc: "" +# -- Configuration for the query-frontend +queryFrontend: + # -- Number of replicas for the query-frontend + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + autoscaling: + # -- Enable autoscaling for the query-frontend + enabled: false + # -- Minimum autoscaling replicas for the query-frontend + minReplicas: 1 + # -- Maximum autoscaling replicas for the query-frontend + maxReplicas: 3 + # -- Target CPU utilisation percentage for the query-frontend + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the query-frontend + targetMemoryUtilizationPercentage: null + # -- Allows one to define custom metrics using the HPA/v2 schema (for example, Pods, Object or External metrics) + customMetrics: [] + # - type: Pods + # pods: + # metric: + # name: loki_query_rate + # target: + # type: AverageValue + # averageValue: 100 + behavior: + # -- Enable autoscaling behaviours + enabled: false + # -- define scale down policies, must conform to HPAScalingRules + scaleDown: {} + # -- define scale up policies, must conform to HPAScalingRules + scaleUp: {} + image: + # -- The Docker registry for the query-frontend image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the query-frontend image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the query-frontend image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for query-frontend pods + priorityClassName: null + # -- Labels for query-frontend pods + podLabels: {} + # -- Annotations for query-frontend pods + podAnnotations: {} + # -- Labels for query-frontend service + serviceLabels: {} + # -- Annotations for query-frontend service + serviceAnnotations: {} + # -- Additional CLI args for the query-frontend + extraArgs: [] + # -- Environment variables to add to the query-frontend pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the query-frontend pods + extraEnvFrom: [] + # -- Volume mounts to add to the query-frontend pods + extraVolumeMounts: [] + # -- Volumes to add to the query-frontend pods + extraVolumes: [] + # -- Resource requests and limits for the query-frontend + resources: {} + # -- Containers to add to the query-frontend pods + extraContainers: [] + # -- Grace period to allow the query-frontend to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for query-frontend pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: query-frontend + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Node selector for query-frontend pods + nodeSelector: {} + # -- Topology Spread Constraints for query-frontend pods + topologySpreadConstraints: [] + # -- Tolerations for query-frontend pods + tolerations: [] + # -- Adds the appProtocol field to the queryFrontend service. This allows queryFrontend to work with istio protocol selection. + appProtocol: + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + grpc: "" +# -- Configuration for the query-scheduler +queryScheduler: + # -- Number of replicas for the query-scheduler. + # It should be lower than `-querier.max-concurrent` to avoid generating back-pressure in queriers; + # it's also recommended that this value evenly divides the latter + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the query-scheduler image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the query-scheduler image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the query-scheduler image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for query-scheduler pods + priorityClassName: null + # -- Labels for query-scheduler pods + podLabels: {} + # -- Annotations for query-scheduler pods + podAnnotations: {} + # -- Labels for query-scheduler service + serviceLabels: {} + # -- Annotations for query-scheduler service + serviceAnnotations: {} + # -- Additional CLI args for the query-scheduler + extraArgs: [] + # -- Environment variables to add to the query-scheduler pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the query-scheduler pods + extraEnvFrom: [] + # -- Volume mounts to add to the query-scheduler pods + extraVolumeMounts: [] + # -- Volumes to add to the query-scheduler pods + extraVolumes: [] + # -- Resource requests and limits for the query-scheduler + resources: {} + # -- Containers to add to the query-scheduler pods + extraContainers: [] + # -- Grace period to allow the query-scheduler to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for query-scheduler pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: query-scheduler + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: 1 + # -- Node selector for query-scheduler pods + nodeSelector: {} + # -- Topology Spread Constraints for query-scheduler pods + topologySpreadConstraints: [] + # -- Tolerations for query-scheduler pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" +# -- Configuration for the index-gateway +indexGateway: + # -- Number of replicas for the index-gateway + replicas: 0 + # -- Whether the index gateway should join the memberlist hashring + joinMemberlist: true + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the index-gateway image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the index-gateway image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the index-gateway image. Overrides `loki.image.tag` + tag: null + # -- The name of the PriorityClass for index-gateway pods + priorityClassName: null + # -- Labels for index-gateway pods + podLabels: {} + # -- Annotations for index-gateway pods + podAnnotations: {} + # -- Labels for index-gateway service + serviceLabels: {} + # -- Annotations for index-gateway service + serviceAnnotations: {} + # -- Additional CLI args for the index-gateway + extraArgs: [] + # -- Environment variables to add to the index-gateway pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the index-gateway pods + extraEnvFrom: [] + # -- Volume mounts to add to the index-gateway pods + extraVolumeMounts: [] + # -- Volumes to add to the index-gateway pods + extraVolumes: [] + # -- Resource requests and limits for the index-gateway + resources: {} + # -- Containers to add to the index-gateway pods + extraContainers: [] + # -- Init containers to add to the index-gateway pods + initContainers: [] + # -- Grace period to allow the index-gateway to shutdown before it is killed. + terminationGracePeriodSeconds: 300 + # -- Affinity for index-gateway pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: index-gateway + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Node selector for index-gateway pods + nodeSelector: {} + # -- Topology Spread Constraints for index-gateway pods + topologySpreadConstraints: [] + # -- Tolerations for index-gateway pods + tolerations: [] + persistence: + # -- Enable creating PVCs which is required when using boltdb-shipper + enabled: false + # -- Use emptyDir with ramdisk for storage. **Please note that all data in indexGateway will be lost on pod restart** + inMemory: false + # -- Size of persistent or memory disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for index gateway PVCs + annotations: {} + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + # -- UpdateStrategy for the indexGateway StatefulSet. + updateStrategy: + # -- One of 'OnDelete' or 'RollingUpdate' + type: RollingUpdate + # -- Optional for updateStrategy.type=RollingUpdate. See [Partitioned rolling updates](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) in the StatefulSet docs for details. + # rollingUpdate: + # partition: 0 +# -- Configuration for the compactor +compactor: + # -- Number of replicas for the compactor + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the compactor image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the compactor image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the compactor image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for compactor pods + priorityClassName: null + # -- Labels for compactor pods + podLabels: {} + # -- Annotations for compactor pods + podAnnotations: {} + # -- Affinity for compactor pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: compactor + topologyKey: kubernetes.io/hostname + # -- Labels for compactor service + serviceLabels: {} + # -- Annotations for compactor service + serviceAnnotations: {} + # -- Additional CLI args for the compactor + extraArgs: [] + # -- Environment variables to add to the compactor pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the compactor pods + extraEnvFrom: [] + # -- Volume mounts to add to the compactor pods + extraVolumeMounts: [] + # -- Volumes to add to the compactor pods + extraVolumes: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- Resource requests and limits for the compactor + resources: {} + # -- Containers to add to the compactor pods + extraContainers: [] + # -- Init containers to add to the compactor pods + initContainers: [] + # -- Grace period to allow the compactor to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Node selector for compactor pods + nodeSelector: {} + # -- Tolerations for compactor pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + persistence: + # -- Enable creating PVCs for the compactor + enabled: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for compactor PVCs + annotations: {} + # -- List of the compactor PVCs + # @notationType -- list + claims: + - name: data + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # - name: wal + # size: 150Gi + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + serviceAccount: + create: false + # -- The name of the ServiceAccount to use for the compactor. + # If not set and create is true, a name is generated by appending + # "-compactor" to the common ServiceAccount. + name: null + # -- Image pull secrets for the compactor service account + imagePullSecrets: [] + # -- Annotations for the compactor service account + annotations: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true +# -- Configuration for the bloom-gateway +bloomGateway: + # -- Number of replicas for the bloom-gateway + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the bloom-gateway image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the bloom-gateway image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the bloom-gateway image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for bloom-gateway pods + priorityClassName: null + # -- Labels for bloom-gateway pods + podLabels: {} + # -- Annotations for bloom-gateway pods + podAnnotations: {} + # -- Affinity for bloom-gateway pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: bloom-gateway + topologyKey: kubernetes.io/hostname + # -- Labels for bloom-gateway service + serviceLabels: {} + # -- Annotations for bloom-gateway service + serviceAnnotations: {} + # -- Additional CLI args for the bloom-gateway + extraArgs: [] + # -- Environment variables to add to the bloom-gateway pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the bloom-gateway pods + extraEnvFrom: [] + # -- Volume mounts to add to the bloom-gateway pods + extraVolumeMounts: [] + # -- Volumes to add to the bloom-gateway pods + extraVolumes: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- Resource requests and limits for the bloom-gateway + resources: {} + # -- Containers to add to the bloom-gateway pods + extraContainers: [] + # -- Init containers to add to the bloom-gateway pods + initContainers: [] + # -- Grace period to allow the bloom-gateway to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Node selector for bloom-gateway pods + nodeSelector: {} + # -- Tolerations for bloom-gateway pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + persistence: + # -- Enable creating PVCs for the bloom-gateway + enabled: false + # -- Annotations for bloom-gateway PVCs + annotations: {} + # -- List of the bloom-gateway PVCs + # @notationType -- list + claims: + - name: data + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + serviceAccount: + create: false + # -- The name of the ServiceAccount to use for the bloom-gateway. + # If not set and create is true, a name is generated by appending + # "-bloom-gateway" to the common ServiceAccount. + name: null + # -- Image pull secrets for the bloom-gateway service account + imagePullSecrets: [] + # -- Annotations for the bloom-gateway service account + annotations: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true +# -- Configuration for the bloom-planner +bloomPlanner: + # -- Number of replicas for the bloom-planner + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the bloom-planner image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the bloom-planner image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the bloom-planner image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for bloom-planner pods + priorityClassName: null + # -- Labels for bloom-planner pods + podLabels: {} + # -- Annotations for bloom-planner pods + podAnnotations: {} + # -- Affinity for bloom-planner pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: bloom-planner + topologyKey: kubernetes.io/hostname + # -- Labels for bloom-planner service + serviceLabels: {} + # -- Annotations for bloom-planner service + serviceAnnotations: {} + # -- Additional CLI args for the bloom-planner + extraArgs: [] + # -- Environment variables to add to the bloom-planner pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the bloom-planner pods + extraEnvFrom: [] + # -- Volume mounts to add to the bloom-planner pods + extraVolumeMounts: [] + # -- Volumes to add to the bloom-planner pods + extraVolumes: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- Resource requests and limits for the bloom-planner + resources: {} + # -- Containers to add to the bloom-planner pods + extraContainers: [] + # -- Init containers to add to the bloom-planner pods + initContainers: [] + # -- Grace period to allow the bloom-planner to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Node selector for bloom-planner pods + nodeSelector: {} + # -- Tolerations for bloom-planner pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + persistence: + # -- Enable creating PVCs for the bloom-planner + enabled: false + # -- Annotations for bloom-planner PVCs + annotations: {} + # -- List of the bloom-planner PVCs + # @notationType -- list + claims: + - name: data + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + serviceAccount: + create: false + # -- The name of the ServiceAccount to use for the bloom-planner. + # If not set and create is true, a name is generated by appending + # "-bloom-planner" to the common ServiceAccount. + name: null + # -- Image pull secrets for the bloom-planner service account + imagePullSecrets: [] + # -- Annotations for the bloom-planner service account + annotations: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true +# -- Configuration for the bloom-builder +bloomBuilder: + # -- Number of replicas for the bloom-builder + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + autoscaling: + # -- Enable autoscaling for the bloom-builder + enabled: false + # -- Minimum autoscaling replicas for the bloom-builder + minReplicas: 1 + # -- Maximum autoscaling replicas for the bloom-builder + maxReplicas: 3 + # -- Target CPU utilisation percentage for the bloom-builder + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for the bloom-builder + targetMemoryUtilizationPercentage: null + # -- Allows one to define custom metrics using the HPA/v2 schema (for example, Pods, Object or External metrics) + customMetrics: [] + # - type: Pods + # pods: + # metric: + # name: loki_query_rate + # target: + # type: AverageValue + # averageValue: 100 + behavior: + # -- Enable autoscaling behaviours + enabled: false + # -- define scale down policies, must conform to HPAScalingRules + scaleDown: {} + # -- define scale up policies, must conform to HPAScalingRules + scaleUp: {} + image: + # -- The Docker registry for the bloom-builder image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the bloom-builder image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the bloom-builder image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for bloom-builder pods + priorityClassName: null + # -- Labels for bloom-builder pods + podLabels: {} + # -- Annotations for bloom-builder pods + podAnnotations: {} + # -- Labels for bloom-builder service + serviceLabels: {} + # -- Annotations for bloom-builder service + serviceAnnotations: {} + # -- Additional CLI args for the bloom-builder + extraArgs: [] + # -- Environment variables to add to the bloom-builder pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the bloom-builder pods + extraEnvFrom: [] + # -- Volume mounts to add to the bloom-builder pods + extraVolumeMounts: [] + # -- Volumes to add to the bloom-builder pods + extraVolumes: [] + # -- Resource requests and limits for the bloom-builder + resources: {} + # -- Containers to add to the bloom-builder pods + extraContainers: [] + # -- Grace period to allow the bloom-builder to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for bloom-builder pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: bloom-builder + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Node selector for bloom-builder pods + nodeSelector: {} + # -- Tolerations for bloom-builder pods + tolerations: [] + # -- Adds the appProtocol field to the queryFrontend service. This allows bloomBuilder to work with istio protocol selection. + appProtocol: + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + grpc: "" +# -- Configuration for the pattern ingester +patternIngester: + # -- Number of replicas for the pattern ingester + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the pattern ingester image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the pattern ingester image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the pattern ingester image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for pattern ingester pods + priorityClassName: null + # -- Labels for pattern ingester pods + podLabels: {} + # -- Annotations for pattern ingester pods + podAnnotations: {} + # -- Affinity for pattern ingester pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: pattern-ingester + topologyKey: kubernetes.io/hostname + # -- Labels for pattern ingester service + serviceLabels: {} + # -- Annotations for pattern ingester service + serviceAnnotations: {} + # -- Additional CLI args for the pattern ingester + extraArgs: [] + # -- Environment variables to add to the pattern ingester pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the pattern ingester pods + extraEnvFrom: [] + # -- Volume mounts to add to the pattern ingester pods + extraVolumeMounts: [] + # -- Volumes to add to the pattern ingester pods + extraVolumes: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- Resource requests and limits for the pattern ingester + resources: {} + # -- Containers to add to the pattern ingester pods + extraContainers: [] + # -- Init containers to add to the pattern ingester pods + initContainers: [] + # -- Grace period to allow the pattern ingester to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Node selector for pattern ingester pods + nodeSelector: {} + # -- Topology Spread Constraints for pattern ingester pods + topologySpreadConstraints: [] + # -- Tolerations for pattern ingester pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + persistence: + # -- Enable creating PVCs for the pattern ingester + enabled: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for pattern ingester PVCs + annotations: {} + # -- List of the pattern ingester PVCs + # @notationType -- list + claims: + - name: data + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # - name: wal + # size: 150Gi + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + serviceAccount: + create: false + # -- The name of the ServiceAccount to use for the pattern ingester. + # If not set and create is true, a name is generated by appending + # "-pattern-ingester" to the common ServiceAccount. + name: null + # -- Image pull secrets for the pattern ingester service account + imagePullSecrets: [] + # -- Annotations for the pattern ingester service account + annotations: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true +# -- Configuration for the ruler +ruler: + # -- The ruler component is optional and can be disabled if desired. + enabled: true + # -- Number of replicas for the ruler + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the ruler image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the ruler image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the ruler image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for ruler pods + priorityClassName: null + # -- Labels for compactor pods + podLabels: {} + # -- Annotations for ruler pods + podAnnotations: {} + # -- Labels for ruler service + serviceLabels: {} + # -- Annotations for ruler service + serviceAnnotations: {} + # -- Additional CLI args for the ruler + extraArgs: [] + # -- Environment variables to add to the ruler pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the ruler pods + extraEnvFrom: [] + # -- Volume mounts to add to the ruler pods + extraVolumeMounts: [] + # -- Volumes to add to the ruler pods + extraVolumes: [] + # -- Resource requests and limits for the ruler + resources: {} + # -- Containers to add to the ruler pods + extraContainers: [] + # -- Init containers to add to the ruler pods + initContainers: [] + # -- Grace period to allow the ruler to shutdown before it is killed + terminationGracePeriodSeconds: 300 + # -- Affinity for ruler pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: ruler + topologyKey: kubernetes.io/hostname + # -- Pod Disruption Budget maxUnavailable + maxUnavailable: null + # -- Node selector for ruler pods + nodeSelector: {} + # -- Topology Spread Constraints for ruler pods + topologySpreadConstraints: [] + # -- Tolerations for ruler pods + tolerations: [] + # -- DNSConfig for ruler pods + dnsConfig: {} + persistence: + # -- Enable creating PVCs which is required when using recording rules + enabled: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for ruler PVCs + annotations: {} + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + # -- Directories containing rules files + directories: {} + # tenant_foo: + # rules1.txt: | + # groups: + # - name: should_fire + # rules: + # - alert: HighPercentageError + # expr: | + # sum(rate({app="foo", env="production"} |= "error" [5m])) by (job) + # / + # sum(rate({app="foo", env="production"}[5m])) by (job) + # > 0.05 + # for: 10m + # labels: + # severity: warning + # annotations: + # summary: High error rate + # - name: credentials_leak + # rules: + # - alert: http-credentials-leaked + # annotations: + # message: "{{ $labels.job }} is leaking http basic auth credentials." + # expr: 'sum by (cluster, job, pod) (count_over_time({namespace="prod"} |~ "http(s?)://(\\w+):(\\w+)@" [5m]) > 0)' + # for: 10m + # labels: + # severity: critical + # rules2.txt: | + # groups: + # - name: example + # rules: + # - alert: HighThroughputLogStreams + # expr: sum by(container) (rate({job=~"loki-dev/.*"}[1m])) > 1000 + # for: 2m + # tenant_bar: + # rules1.txt: | + # groups: + # - name: should_fire + # rules: + # - alert: HighPercentageError + # expr: | + # sum(rate({app="foo", env="production"} |= "error" [5m])) by (job) + # / + # sum(rate({app="foo", env="production"}[5m])) by (job) + # > 0.05 + # for: 10m + # labels: + # severity: warning + # annotations: + # summary: High error rate + # - name: credentials_leak + # rules: + # - alert: http-credentials-leaked + # annotations: + # message: "{{ $labels.job }} is leaking http basic auth credentials." + # expr: 'sum by (cluster, job, pod) (count_over_time({namespace="prod"} |~ "http(s?)://(\\w+):(\\w+)@" [5m]) > 0)' + # for: 10m + # labels: + # severity: critical + # rules2.txt: | + # groups: + # - name: example + # rules: + # - alert: HighThroughputLogStreams + # expr: sum by(container) (rate({job=~"loki-dev/.*"}[1m])) > 1000 + # for: 2m +memcached: + image: + # -- Memcached Docker image repository + repository: memcached + # -- Memcached Docker image tag + tag: 1.6.32-alpine + # -- Memcached Docker image pull policy + pullPolicy: IfNotPresent + # -- The SecurityContext override for memcached pods + podSecurityContext: + runAsNonRoot: true + runAsUser: 11211 + runAsGroup: 11211 + fsGroup: 11211 + # -- The name of the PriorityClass for memcached pods + priorityClassName: null + # -- The SecurityContext for memcached containers + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + allowPrivilegeEscalation: false +memcachedExporter: + # -- Whether memcached metrics should be exported + enabled: true + image: + repository: prom/memcached-exporter + tag: v0.15.0 + pullPolicy: IfNotPresent + resources: + requests: {} + limits: {} + # -- The SecurityContext for memcached exporter containers + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + allowPrivilegeEscalation: false + # -- Extra args to add to the exporter container. + # Example: + # extraArgs: + # memcached.tls.enable: true + # memcached.tls.cert-file: /certs/cert.crt + # memcached.tls.key-file: /certs/cert.key + # memcached.tls.ca-file: /certs/ca.crt + # memcached.tls.insecure-skip-verify: false + # memcached.tls.server-name: memcached + extraArgs: {} +resultsCache: + # -- Specifies whether memcached based results-cache should be enabled + enabled: true + # -- Specify how long cached results should be stored in the results-cache before being expired + defaultValidity: 12h + # -- Memcached operation timeout + timeout: 500ms + # -- Total number of results-cache replicas + replicas: 1 + # -- Port of the results-cache service + port: 11211 + # -- Amount of memory allocated to results-cache for object storage (in MB). + allocatedMemory: 1024 + # -- Maximum item results-cache for memcached (in MB). + maxItemMemory: 5 + # -- Maximum number of connections allowed + connectionLimit: 16384 + # -- Max memory to use for cache write back + writebackSizeLimit: 500MB + # -- Max number of objects to use for cache write back + writebackBuffer: 500000 + # -- Number of parallel threads for cache write back + writebackParallelism: 1 + # -- Extra init containers for results-cache pods + initContainers: [] + # -- Annotations for the results-cache pods + annotations: {} + # -- Node selector for results-cache pods + nodeSelector: {} + # -- Affinity for results-cache pods + affinity: {} + # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints. + # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services. + topologySpreadConstraints: [] + # maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + # -- Tolerations for results-cache pods + tolerations: [] + # -- Pod Disruption Budget + podDisruptionBudget: + maxUnavailable: 1 + # -- The name of the PriorityClass for results-cache pods + priorityClassName: null + # -- Labels for results-cache pods + podLabels: {} + # -- Annotations for results-cache pods + podAnnotations: {} + # -- Management policy for results-cache pods + podManagementPolicy: Parallel + # -- Grace period to allow the results-cache to shutdown before it is killed + terminationGracePeriodSeconds: 60 + # -- Stateful results-cache strategy + statefulStrategy: + type: RollingUpdate + # -- Add extended options for results-cache memcached container. The format is the same as for the memcached -o/--extend flag. + # Example: + # extraExtendedOptions: 'tls,modern,track_sizes' + extraExtendedOptions: "" + # -- Additional CLI args for results-cache + extraArgs: {} + # -- Additional containers to be added to the results-cache pod. + extraContainers: [] + # -- Additional volumes to be added to the results-cache pod (applies to both memcached and exporter containers). + # Example: + # extraVolumes: + # - name: extra-volume + # secret: + # secretName: extra-volume-secret + extraVolumes: [] + # -- Additional volume mounts to be added to the results-cache pod (applies to both memcached and exporter containers). + # Example: + # extraVolumeMounts: + # - name: extra-volume + # mountPath: /etc/extra-volume + # readOnly: true + extraVolumeMounts: [] + # -- Resource requests and limits for the results-cache + # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)). + resources: null + # -- Service annotations and labels + service: + annotations: {} + labels: {} + # -- Persistence settings for the results-cache + persistence: + # -- Enable creating PVCs for the results-cache + enabled: false + # -- Size of persistent disk, must be in G or Gi + storageSize: 10G + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Volume mount path + mountPath: /data +chunksCache: + # -- Specifies whether memcached based chunks-cache should be enabled + enabled: true + # -- Batchsize for sending and receiving chunks from chunks cache + batchSize: 4 + # -- Parallel threads for sending and receiving chunks from chunks cache + parallelism: 5 + # -- Memcached operation timeout + timeout: 2000ms + # -- Specify how long cached chunks should be stored in the chunks-cache before being expired + defaultValidity: 0s + # -- Total number of chunks-cache replicas + replicas: 1 + # -- Port of the chunks-cache service + port: 11211 + # -- Amount of memory allocated to chunks-cache for object storage (in MB). + allocatedMemory: 8192 + # -- Maximum item memory for chunks-cache (in MB). + maxItemMemory: 5 + # -- Maximum number of connections allowed + connectionLimit: 16384 + # -- Max memory to use for cache write back + writebackSizeLimit: 500MB + # -- Max number of objects to use for cache write back + writebackBuffer: 500000 + # -- Number of parallel threads for cache write back + writebackParallelism: 1 + # -- Extra init containers for chunks-cache pods + initContainers: [] + # -- Annotations for the chunks-cache pods + annotations: {} + # -- Node selector for chunks-cache pods + nodeSelector: {} + # -- Affinity for chunks-cache pods + affinity: {} + # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints. + # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services. + topologySpreadConstraints: [] + # maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + # -- Tolerations for chunks-cache pods + tolerations: [] + # -- Pod Disruption Budget + podDisruptionBudget: + maxUnavailable: 1 + # -- The name of the PriorityClass for chunks-cache pods + priorityClassName: null + # -- Labels for chunks-cache pods + podLabels: {} + # -- Annotations for chunks-cache pods + podAnnotations: {} + # -- Management policy for chunks-cache pods + podManagementPolicy: Parallel + # -- Grace period to allow the chunks-cache to shutdown before it is killed + terminationGracePeriodSeconds: 60 + # -- Stateful chunks-cache strategy + statefulStrategy: + type: RollingUpdate + # -- Add extended options for chunks-cache memcached container. The format is the same as for the memcached -o/--extend flag. + # Example: + # extraExtendedOptions: 'tls,no_hashexpand' + extraExtendedOptions: "" + # -- Additional CLI args for chunks-cache + extraArgs: {} + # -- Additional containers to be added to the chunks-cache pod. + extraContainers: [] + # -- Additional volumes to be added to the chunks-cache pod (applies to both memcached and exporter containers). + # Example: + # extraVolumes: + # - name: extra-volume + # secret: + # secretName: extra-volume-secret + extraVolumes: [] + # -- Additional volume mounts to be added to the chunks-cache pod (applies to both memcached and exporter containers). + # Example: + # extraVolumeMounts: + # - name: extra-volume + # mountPath: /etc/extra-volume + # readOnly: true + extraVolumeMounts: [] + # -- Resource requests and limits for the chunks-cache + # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)). + resources: null + # -- Service annotations and labels + service: + annotations: {} + labels: {} + # -- Persistence settings for the chunks-cache + persistence: + # -- Enable creating PVCs for the chunks-cache + enabled: false + # -- Size of persistent disk, must be in G or Gi + storageSize: 10G + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Volume mount path + mountPath: /data +###################################################################################################################### +# +# Subchart configurations +# +###################################################################################################################### +# -- Setting for the Grafana Rollout Operator https://github.com/grafana/helm-charts/tree/main/charts/rollout-operator +rollout_operator: + enabled: false + # -- podSecurityContext is the pod security context for the rollout operator. + # When installing on OpenShift, override podSecurityContext settings with + # + # rollout_operator: + # podSecurityContext: + # fsGroup: null + # runAsGroup: null + # runAsUser: null + podSecurityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + # Set the container security context + securityContext: + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + allowPrivilegeEscalation: false +# -- Configuration for the minio subchart +minio: + enabled: false + replicas: 1 + # Minio requires 2 to 16 drives for erasure code (drivesPerNode * replicas) + # https://docs.min.io/docs/minio-erasure-code-quickstart-guide + # Since we only have 1 replica, that means 2 drives must be used. + drivesPerNode: 2 + rootUser: enterprise-logs + rootPassword: supersecret + buckets: + - name: chunks + policy: none + purge: false + - name: ruler + policy: none + purge: false + - name: admin + policy: none + purge: false + persistence: + size: 5Gi + annotations: {} + resources: + requests: + cpu: 100m + memory: 128Mi + # Allow the address used by Loki to refer to Minio to be overridden + address: null +# Create extra manifests via values. Would be passed through `tpl` for templating +# objects can also be provided as multiline strings, useful for templating field names +extraObjects: [] +# - apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: loki-alerting-rules +# data: +# loki-alerting-rules.yaml: |- +# groups: +# - name: example +# rules: +# - alert: example +# expr: | +# sum(count_over_time({app="loki"} |~ "error")) > 0 +# for: 3m +# labels: +# severity: warning +# category: logs +# annotations: +# message: "loki has encountered errors" +# - | +# apiVersion: v1 +# kind: Secret +# type: Opaque +# metadata: +# name: loki-distributed-basic-auth +# data: +# {{- range .Values.loki.tenants }} +# {{ .name }}: {{ b64enc .password | quote }} +# {{- end }} + +sidecar: + image: + # -- The Docker registry and image for the k8s sidecar + repository: kiwigrid/k8s-sidecar + # -- Docker image tag + tag: 1.28.0 + # -- Docker image sha. If empty, no sha will be used + sha: "" + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Resource requests and limits for the sidecar + resources: {} + # limits: + # cpu: 100m + # memory: 100Mi + # requests: + # cpu: 50m + # memory: 50Mi + # -- The SecurityContext for the sidecar. + securityContext: {} + # -- Set to true to skip tls verification for kube api calls. + skipTlsVerify: false + # -- Ensure that rule files aren't conflicting and being overwritten by prefixing their name with the namespace they are defined in. + enableUniqueFilenames: false + # -- Readiness probe definition. Probe is disabled on the sidecar by default. + readinessProbe: {} + # -- Liveness probe definition. Probe is disabled on the sidecar by default. + livenessProbe: {} + rules: + # -- Whether or not to create a sidecar to ingest rule from specific ConfigMaps and/or Secrets. + enabled: true + # -- Label that the configmaps/secrets with rules will be marked with. + label: loki_rule + # -- Label value that the configmaps/secrets with rules will be set to. + labelValue: "" + # -- Folder into which the rules will be placed. + folder: /rules + # -- Comma separated list of namespaces. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify 'ALL' to search in all namespaces. + searchNamespace: null + # -- Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH request, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # -- Search in configmap, secret, or both. + resource: both + # -- Absolute path to the shell script to execute after a configmap or secret has been reloaded. + script: null + # -- WatchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S. + watchServerTimeout: 60 + # + # -- WatchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # Defaults to 66sec. + watchClientTimeout: 60 + # -- Log level of the sidecar container. + logLevel: INFO +############################################## WARNING ############################################################### +# +# DEPRECATED VALUES +# +# The following values are deprecated and will be removed in a future version of the helm chart! +# +############################################## WARNING ############################################################## + +# -- DEPRECATED Monitoring section determines which monitoring features to enable, this section is being replaced +# by https://github.com/grafana/meta-monitoring-chart +monitoring: + # Dashboards for monitoring Loki + dashboards: + # -- If enabled, create configmap with dashboards for monitoring Loki + enabled: false + # -- Alternative namespace to create dashboards ConfigMap in + namespace: null + # -- Additional annotations for the dashboards ConfigMap + annotations: {} + # -- Labels for the dashboards ConfigMap + labels: + grafana_dashboard: "1" + # -- DEPRECATED Recording rules for monitoring Loki, required for some dashboards + rules: + # -- If enabled, create PrometheusRule resource with Loki recording rules + enabled: false + # -- Include alerting rules + alerting: true + # -- Specify which individual alerts should be disabled + # -- Instead of turning off each alert one by one, set the .monitoring.rules.alerting value to false instead. + # -- If you disable all the alerts and keep .monitoring.rules.alerting set to true, the chart will fail to render. + disabled: {} + # LokiRequestErrors: true + # LokiRequestPanics: true + # -- Alternative namespace to create PrometheusRule resources in + namespace: null + # -- Additional annotations for the rules PrometheusRule resource + annotations: {} + # -- Additional labels for the rules PrometheusRule resource + labels: {} + # -- Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + # -- Additional groups to add to the rules file + additionalGroups: [] + # - name: additional-loki-rules + # rules: + # - record: job:loki_request_duration_seconds_bucket:sum_rate + # expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job) + # - record: job_route:loki_request_duration_seconds_bucket:sum_rate + # expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route) + # - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate + # expr: sum(rate(container_cpu_usage_seconds_total[1m])) by (node, namespace, pod, container) + # -- DEPRECATED ServiceMonitor configuration + serviceMonitor: + # -- If enabled, ServiceMonitor resources for Prometheus Operator are created + enabled: false + # -- Namespace selector for ServiceMonitor resources + namespaceSelector: {} + # -- ServiceMonitor annotations + annotations: {} + # -- Additional ServiceMonitor labels + labels: {} + # -- ServiceMonitor scrape interval + # Default is 15s because included recording rules use a 1m rate, and scrape interval needs to be at + # least 1/4 rate interval. + interval: 15s + # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s) + scrapeTimeout: null + # -- ServiceMonitor relabel configs to apply to samples before scraping + # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + relabelings: [] + # -- ServiceMonitor metric relabel configs to apply to samples before ingestion + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + metricRelabelings: [] + # -- ServiceMonitor will use http by default, but you can pick https as well + scheme: http + # -- ServiceMonitor will use these tlsConfig settings to make the health check requests + tlsConfig: null + # -- If defined, will create a MetricsInstance for the Grafana Agent Operator. + metricsInstance: + # -- If enabled, MetricsInstance resources for Grafana Agent Operator are created + enabled: true + # -- MetricsInstance annotations + annotations: {} + # -- Additional MetricsInstance labels + labels: {} + # -- If defined a MetricsInstance will be created to remote write metrics. + remoteWrite: null + # -- DEPRECATED Self monitoring determines whether Loki should scrape its own logs. + # This feature currently relies on the Grafana Agent Operator being installed, + # which is installed by default using the grafana-agent-operator sub-chart. + # It will create custom resources for GrafanaAgent, LogsInstance, and PodLogs to configure + # scrape configs to scrape its own logs with the labels expected by the included dashboards. + selfMonitoring: + enabled: false + # -- Tenant to use for self monitoring + tenant: + # -- Name of the tenant + name: "self-monitoring" + # -- Password of the gateway for Basic auth + password: null + # -- Namespace to create additional tenant token secret in. Useful if your Grafana instance + # is in a separate namespace. Token will still be created in the canary namespace. + secretNamespace: "{{ .Release.Namespace }}" + # -- DEPRECATED Grafana Agent configuration + grafanaAgent: + # -- DEPRECATED Controls whether to install the Grafana Agent Operator and its CRDs. + # Note that helm will not install CRDs if this flag is enabled during an upgrade. + # In that case install the CRDs manually from https://github.com/grafana/agent/tree/main/production/operator/crds + installOperator: false + # -- Grafana Agent annotations + annotations: {} + # -- Additional Grafana Agent labels + labels: {} + # -- Enable the config read api on port 8080 of the agent + enableConfigReadAPI: false + # -- The name of the PriorityClass for GrafanaAgent pods + priorityClassName: null + # -- Resource requests and limits for the grafanaAgent pods + resources: {} + # limits: + # memory: 200Mi + # requests: + # cpu: 50m + # memory: 100Mi + # -- Tolerations for GrafanaAgent pods + tolerations: [] + # PodLogs configuration + podLogs: + # -- PodLogs version + apiVersion: monitoring.grafana.com/v1alpha1 + # -- PodLogs annotations + annotations: {} + # -- Additional PodLogs labels + labels: {} + # -- PodLogs relabel configs to apply to samples before scraping + # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + relabelings: [] + # -- Additional pipeline stages to process logs after scraping + # https://grafana.com/docs/agent/latest/operator/api/#pipelinestagespec-a-namemonitoringgrafanacomv1alpha1pipelinestagespeca + additionalPipelineStages: [] + # LogsInstance configuration + logsInstance: + # -- LogsInstance annotations + annotations: {} + # -- Additional LogsInstance labels + labels: {} + # -- Additional clients for remote write + clients: null +# -- DEPRECATED Configuration for the table-manager. The table-manager is only necessary when using a deprecated +# index type such as Cassandra, Bigtable, or DynamoDB, it has not been necessary since loki introduced self- +# contained index types like 'boltdb-shipper' and 'tsdb'. This will be removed in a future helm chart. +tableManager: + # -- Specifies whether the table-manager should be enabled + enabled: false + image: + # -- The Docker registry for the table-manager image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the table-manager image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the table-manager image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for table-manager pods + priorityClassName: null + # -- Labels for table-manager pods + podLabels: {} + # -- Annotations for table-manager deployment + annotations: {} + # -- Annotations for table-manager pods + podAnnotations: {} + service: + # -- Annotations for table-manager Service + annotations: {} + # -- Additional labels for table-manager Service + labels: {} + # -- Additional CLI args for the table-manager + extraArgs: [] + # -- Environment variables to add to the table-manager pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the table-manager pods + extraEnvFrom: [] + # -- Volume mounts to add to the table-manager pods + extraVolumeMounts: [] + # -- Volumes to add to the table-manager pods + extraVolumes: [] + # -- Resource requests and limits for the table-manager + resources: {} + # -- Containers to add to the table-manager pods + extraContainers: [] + # -- Grace period to allow the table-manager to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for table-manager pods. + # @default -- Hard node and anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: table-manager + topologyKey: kubernetes.io/hostname + # -- DNS config table-manager pods + dnsConfig: {} + # -- Node selector for table-manager pods + nodeSelector: {} + # -- Tolerations for table-manager pods + tolerations: [] + # -- Enable deletes by retention + retention_deletes_enabled: false + # -- Set retention period + retention_period: 0 diff --git a/opencloud/templates/oc-auth/deployment.yaml b/opencloud/templates/oc-auth/deployment.yaml index d3c722e..23b44d8 100644 --- a/opencloud/templates/oc-auth/deployment.yaml +++ b/opencloud/templates/oc-auth/deployment.yaml @@ -26,7 +26,7 @@ spec: containers: - image: "{{ .Values.ocAuth.image }}" name: oc-auth - command: ["tail", "-f", "/dev/null"] + #command: ["tail", "-f", "/dev/null"] volumeMounts: - name: public-key-volume mountPath: /keys/public/public.pem