add tcs log elk
ycz008
parent
5f8486e8ee
commit
fc4dc65b0f
|
|
@ -0,0 +1,203 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: filebeat-config
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
data:
|
||||
filebeat.yml: |-
|
||||
filebeat.config:
|
||||
modules:
|
||||
path: ${path.config}/modules.d/*.yml
|
||||
reload.enabled: false
|
||||
|
||||
# To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
node: ${NODE_NAME}
|
||||
hints.enabled: true
|
||||
# hints.default_config:
|
||||
# type: container
|
||||
# paths:
|
||||
# - /var/log/containers/*${data.kubernetes.container.id}.log
|
||||
|
||||
processors:
|
||||
- add_cloud_metadata: ~
|
||||
|
||||
filebeat.inputs:
|
||||
- type: log
|
||||
paths:
|
||||
- /data/app_logs/*/*/logstash/*.log
|
||||
json.keys_under_root: true
|
||||
tags: ['json']
|
||||
|
||||
#All data to indexed to Elasticsearch
|
||||
output.logstash:
|
||||
hosts: ["logstash.tcs.svc:5044"]
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: filebeat
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
spec:
|
||||
serviceAccountName: filebeat
|
||||
terminationGracePeriodSeconds: 30
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: filebeat
|
||||
image: docker.elastic.co/beats/filebeat:7.16.3
|
||||
args: [
|
||||
"-c", "/etc/filebeat.yml",
|
||||
"-e",
|
||||
]
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
resources:
|
||||
limits:
|
||||
memory: 200Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 100Mi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/filebeat.yml
|
||||
readOnly: true
|
||||
subPath: filebeat.yml
|
||||
- name: data
|
||||
mountPath: /usr/share/filebeat/data
|
||||
- name: app-logs
|
||||
mountPath: /data/app_logs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
defaultMode: 0640
|
||||
name: filebeat-config
|
||||
- name: app-logs
|
||||
hostPath:
|
||||
path: /data/app_logs
|
||||
- name: data
|
||||
hostPath:
|
||||
# When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
|
||||
path: /var/lib/filebeat-data
|
||||
type: DirectoryOrCreate
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: filebeat
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: filebeat
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: filebeat
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: filebeat-kubeadm-config
|
||||
namespace: kube-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: filebeat-kubeadm-config
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: filebeat
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
rules:
|
||||
- apiGroups: [""] # "" indicates the core API group
|
||||
resources:
|
||||
- namespaces
|
||||
- pods
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- watch
|
||||
- list
|
||||
- apiGroups: ["apps"]
|
||||
resources:
|
||||
- replicasets
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: filebeat
|
||||
# should be the namespace where filebeat is running
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
rules:
|
||||
- apiGroups:
|
||||
- coordination.k8s.io
|
||||
resources:
|
||||
- leases
|
||||
verbs: ["get", "create", "update"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: filebeat-kubeadm-config
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- configmaps
|
||||
resourceNames:
|
||||
- kubeadm-config
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kibana-config
|
||||
data:
|
||||
kibana.yml: |-
|
||||
server.name: kibana
|
||||
server.host: 0.0.0.0
|
||||
elasticsearch.hosts: [ "http://elastic:9200" ]
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kibana
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kibana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
containers:
|
||||
- name: kibana
|
||||
image: docker.elastic.co/kibana/kibana:7.16.3
|
||||
ports:
|
||||
- containerPort: 5601
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /usr/share/kibana/config/kibana.yml
|
||||
subPath: kibana.yml
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: kibana-config
|
||||
items:
|
||||
- key: kibana.yml
|
||||
path: kibana.yml
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kibana
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
ports:
|
||||
- port: 5601
|
||||
targetPort: 5601
|
||||
selector:
|
||||
app: kibana
|
||||
type: ClusterIP
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: logstash-config
|
||||
data:
|
||||
logstash.conf: |-
|
||||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if [tags][json] {
|
||||
json {
|
||||
source => "message"
|
||||
}
|
||||
}
|
||||
mutate {
|
||||
split => { "[log][file][path]" => "/" }
|
||||
add_field => { "env" => "%{[log][file][path][3]}" }
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => ["http://elastic:9200"]
|
||||
index => "%{[env]}-logback-beaconfire-%{+YYYY.MM.dd}"
|
||||
}
|
||||
}
|
||||
logstash.yml: |-
|
||||
http.host: "0.0.0.0"
|
||||
xpack.monitoring.elasticsearch.hosts: [ "http://elastic:9200" ]
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: logstash
|
||||
labels:
|
||||
app: logstash
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: logstash
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: logstash
|
||||
spec:
|
||||
containers:
|
||||
- name: logstash
|
||||
image: docker.elastic.co/logstash/logstash:7.16.3
|
||||
ports:
|
||||
- containerPort: 5044
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /usr/share/logstash/pipeline/logstash.conf
|
||||
subPath: logstash.conf
|
||||
readOnly: true
|
||||
- name: config
|
||||
mountPath: /usr/share/logstash/config/logstash.yml
|
||||
subPath: logstash.yml
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: logstash-config
|
||||
items:
|
||||
- key: logstash.conf
|
||||
path: logstash.conf
|
||||
- key: logstash.yml
|
||||
path: logstash.yml
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: logstash
|
||||
labels:
|
||||
app: logstash
|
||||
spec:
|
||||
ports:
|
||||
- port: 5044
|
||||
targetPort: 5044
|
||||
selector:
|
||||
app: logstash
|
||||
type: ClusterIP
|
||||
Loading…
Reference in New Issue