add logstash alert

build-image/logstash/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.elastic.co/logstash/logstash:7.16.3
+
+COPY log-alert.sh /usr/local/bin/
+
+RUN logstash-plugin install logstash-output-exec

build-image/logstash/log-alert.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# @timestamp serviceName env level message
+if (($#!=5));then
+    echo "./log-alert.sh %{@timestamp} %{serviceName} %{env} %{level} %{message}"
+    exit 1
+fi
+
+if [[ -z $1 || -z $2 || -z $3 || -z $4 || -z $5 ]];then
+    echo "one of '%{@timestamp} %{serviceName} %{env} %{level} %{message}' is empty"
+    exit 1
+fi
+
+msg="timestamp: $1\nserviceName: $2\nenv: $3\nlevel: $4\nmessage: $5\n"
+#echo -ne $msg
+
+curl -X POST -H "Content-Type: application/json" -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"$msg\"}}" https://open.feishu.cn/open-apis/bot/v2/hook/29dd52e5-70d5-44b0-a443-22ea85382646

dev-upgrade/elastic/logstash-alert.yaml

@@ -0,0 +1,119 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-config
+data:
+  logstash.conf: |-
+    input {
+      kafka{
+        bootstrap_servers => "10.2.0.12:30002,10.2.0.12:30003,10.2.0.12:30004"
+        topics => ["beaconfire-logback-prod"]
+        group_id => "logstash-app"
+        auto_offset_reset => "latest"
+        codec => json
+      }
+    }
+
+    filter {
+      if [tags][json] {
+        json {
+          source => "message"
+        }
+      }
+      if [level] == "TRACE" {
+        drop {}
+      }
+      if [level] == "DEBUG" {
+        drop {}
+      }
+      if [level] == "INFO" {
+        drop {}
+      }
+      if [level] == "WARN" {
+        drop {}
+      }
+      if [message] =~ "Fail to parse JWT due to: Jwt expired at" {
+        drop {}
+      }
+      mutate {
+        split => { "[log][file][path]" => "/" }
+        add_field => { "env" => "%{[log][file][path][3]}" }
+      }
+      mutate {
+        join => { "[log][file][path]" => "/" }
+      }
+    }
+
+    output {
+      # stdout { codec => json_lines }
+      # file {
+      #   path => "1.json"
+      #   codec => "json_lines"
+      # }
+      exec {
+        command => "/usr/local/bin/log-alert.sh '%{@timestamp}' '%{serviceName}' '%{env}' '%{level}' '%{message}'"
+      }
+    }
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logstash
+  labels:
+    app: logstash
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: logstash
+  template:
+    metadata:
+      labels:
+        app: logstash
+    spec:
+      containers:
+        - name: logstash
+          image: beaconfireiic/logstash:7.16.3
+          ports:
+            - containerPort: 5044
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 1
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /usr/share/logstash/pipeline/logstash.conf
+              subPath: logstash.conf
+              readOnly: true
+            - name: config
+              mountPath: /usr/share/logstash/config/logstash.yml
+              subPath: logstash.yml
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: logstash-config
+            items:
+              - key: logstash.conf
+                path: logstash.conf
+              - key: logstash.yml
+                path: logstash.yml
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: logstash
+  labels:
+    app: logstash
+spec:
+  ports:
+    - port: 5044
+      targetPort: 5044
+  selector:
+    app: logstash
+  type: ClusterIP