add prod prom

prod-upgrade/prom/grafana.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: grafana
+  name: grafana
+spec:
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      securityContext:
+        fsGroup: 472
+        supplementalGroups:
+          - 0
+      containers:
+        - name: grafana
+          image: grafana/grafana:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http-grafana
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /robots.txt
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3000
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: 250m
+              memory: 750Mi
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: grafana-pv
+      volumes:
+        - name: grafana-pv
+          persistentVolumeClaim:
+            claimName: grafana-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+spec:
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+  selector:
+    app: grafana
+  sessionAffinity: None
+  type: ClusterIP

prod-upgrade/prom/prometheus.yaml

@@ -0,0 +1,120 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prom-config
+data:
+  prom-config: |-
+    global:
+      scrape_interval: 15s
+
+    scrape_configs:
+      - job_name: mysql # To get metrics about the mysql exporter’s targets
+        params:
+          auth_module: [client]
+        static_configs:
+          - targets:
+            # All mysql hostnames or unix sockets to monitor.
+              - mysql-0.base.svc:3306
+              - mysql-1.base.svc:3306
+              - mysql-2.base.svc:3306
+              - mysql-3.base.svc:3306
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: __param_target
+          - source_labels: [__param_target]
+            target_label: instance
+          - target_label: __address__
+            replacement: mysqld-exporter.base.svc:9104
+    rule_files:
+      - /etc/prometheus/rules/rule-0.yaml
+    alerting:
+      alert_relabel_configs:
+      alertmanagers:
+  prom-rule-0: |-
+    groups:
+    - name: mysqld_rules
+      rules:
+    
+      # Record slave lag seconds for pre-computed timeseries that takes
+      # `mysql_slave_status_sql_delay` into account
+      - record: instance:mysql_slave_lag_seconds
+        expr: mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay
+    
+      # Record slave lag via heartbeat method
+      - record: instance:mysql_heartbeat_lag_seconds
+        expr: mysql_heartbeat_now_timestamp_seconds - mysql_heartbeat_stored_timestamp_seconds
+    
+      - record: job:mysql_transactions:rate5m
+        expr: sum without (command) (rate(mysql_global_status_commands_total{command=~"(commit|rollback)"}[5m]))
+    
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: prometheus
+spec:
+  replicas: 1
+  serviceName: mysql
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+      - name: prometheus
+        image: prom/prometheus:v2.51.2
+        args:
+        - --config.file=/etc/prometheus/prometheus.yaml
+        - --storage.tsdb.path=/etc/prometheus/data
+        - --storage.tsdb.retention.time=15d
+        volumeMounts:
+        - name: prometheus
+          mountPath: /etc/prometheus/data
+        - name: prom-config
+          mountPath: /etc/prometheus/prometheus.yaml
+          subPath: prometheus.yaml
+          readOnly: true
+        - name: prom-config
+          mountPath: /etc/prometheus/rules/rule-0.yaml
+          subPath: rule-0.yaml
+          readOnly: true
+      volumes:
+      - name: prometheus
+        hostPath:
+          path: /data/prometheus/data
+      - name: prom-config
+        configMap:
+          name: prom-config
+          items:
+          - key: prom-config
+            path: prometheus.yaml
+          - key: prom-rule-0
+            path: rule-0.yaml
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "kubernetes.io/hostname"
+                operator: In
+                values:
+                - bfs-k8snode-10-2-2-7.hetzner.base.beaconfireinc.com
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+spec:
+  ports:
+  - protocol: TCP
+    port: 9090
+    targetPort: 9090
+  selector:
+    app: prometheus
+  type: ClusterIP