diff --git a/prod-upgrade/elastic/filebeat.yaml b/prod-upgrade/elastic/filebeat.yaml new file mode 100644 index 0000000..8250899 --- /dev/null +++ b/prod-upgrade/elastic/filebeat.yaml @@ -0,0 +1,203 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: + filebeat.autodiscover: + providers: + - type: kubernetes + node: ${NODE_NAME} + hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log + + processors: + - add_cloud_metadata: ~ + + filebeat.inputs: + - type: log + paths: + - /data/app_logs/*/logstash/*.log + json.keys_under_root: true + tags: ['json'] + + #All data to indexed to Elasticsearch + output.logstash: + hosts: ["logstash.base.svc:5044"] + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + selector: + matchLabels: + k8s-app: filebeat + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:7.16.3 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: app-logs + mountPath: /data/app_logs + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0640 + name: filebeat-config + - name: app-logs + hostPath: + path: /data/app_logs + - name: data + hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat \ No newline at end of file diff --git a/prod-upgrade/elastic/logstash.yaml b/prod-upgrade/elastic/logstash.yaml new file mode 100644 index 0000000..dd7bbfd --- /dev/null +++ b/prod-upgrade/elastic/logstash.yaml @@ -0,0 +1,95 @@ +# 部署命名空间 base +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstash-config +data: + logstash.conf: |- + input { + beats { + port => 5044 + } + } + + filter { + if [tags][json] { + json { + source => "message" + } + } + } + + output { + stdout { codec => rubydebug } + elasticsearch { + hosts => ["http://10.2.0.12:9200"] + index => "prod-logback-beaconfire-%{+YYYY.MM.dd}" + } + } + logstash.yml: |- + http.host: "0.0.0.0" + xpack.monitoring.elasticsearch.hosts: [ "http://10.2.0.12:9200" ] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logstash + labels: + app: logstash +spec: + replicas: 1 + selector: + matchLabels: + app: logstash + template: + metadata: + labels: + app: logstash + spec: + containers: + - name: logstash + image: docker.elastic.co/logstash/logstash:7.16.3 + ports: + - containerPort: 5044 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 1Gi + volumeMounts: + - name: config + mountPath: /usr/share/logstash/pipeline/logstash.conf + subPath: logstash.conf + readOnly: true + - name: config + mountPath: /usr/share/logstash/config/logstash.yml + subPath: logstash.yml + readOnly: true + volumes: + - name: config + configMap: + name: logstash-config + items: + - key: logstash.conf + path: logstash.conf + - key: logstash.yml + path: logstash.yml + +--- +apiVersion: v1 +kind: Service +metadata: + name: logstash + labels: + app: logstash +spec: + ports: + - port: 5044 + targetPort: 5044 + selector: + app: logstash + type: ClusterIP \ No newline at end of file