diff --git a/build-image/node/Dockerfile b/build-image/node/Dockerfile deleted file mode 100644 index 866d40b..0000000 --- a/build-image/node/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM node:16.17.1 - -MAINTAINER smallkoudai - -ARG VERSION=16.17.1 -RUN npm install --global cross-env && rm -rf ~/.npm diff --git a/trainer-env/kafka/confluent-platform.yaml b/trainer-env/kafka/confluent-platform.yaml new file mode 100644 index 0000000..213bf48 --- /dev/null +++ b/trainer-env/kafka/confluent-platform.yaml @@ -0,0 +1,153 @@ +# deployment +--- +apiVersion: platform.confluent.io/v1beta1 +kind: Zookeeper +metadata: + name: zookeeper + namespace: basic +spec: + replicas: 3 + podTemplate: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - zookeeper + topologyKey: kubernetes.io/hostname + image: + application: confluentinc/cp-zookeeper:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dataVolumeCapacity: 10Gi + logVolumeCapacity: 10Gi + storageClass: + name: kafka-local +--- +apiVersion: platform.confluent.io/v1beta1 +kind: Kafka +metadata: + name: kafka + namespace: basic +spec: + replicas: 3 + podTemplate: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - kafka + topologyKey: kubernetes.io/hostname + image: + application: confluentinc/cp-server:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dataVolumeCapacity: 20Gi + storageClass: + name: kafka-local + metricReporter: + enabled: true + dependencies: + zookeeper: + endpoint: zookeeper.basic.svc.cluster.local:2181 +--- +apiVersion: platform.confluent.io/v1beta1 +kind: Connect +metadata: + name: connect + namespace: basic +spec: + replicas: 1 + image: + application: confluentinc/cp-server-connect:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dependencies: + kafka: + bootstrapEndpoint: kafka:9071 +--- +apiVersion: platform.confluent.io/v1beta1 +kind: KsqlDB +metadata: + name: ksqldb + namespace: basic +spec: + replicas: 1 + image: + application: confluentinc/cp-ksqldb-server:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dataVolumeCapacity: 10Gi + storageClass: + name: kafka-local +--- +apiVersion: platform.confluent.io/v1beta1 +kind: ControlCenter +metadata: + name: controlcenter + namespace: basic +spec: + replicas: 1 + image: + application: confluentinc/cp-enterprise-control-center:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dataVolumeCapacity: 10Gi + storageClass: + name: kafka-local + dependencies: + schemaRegistry: + url: http://schemaregistry.basic.svc.cluster.local:8081 + ksqldb: + - name: ksqldb + url: http://ksqldb.basic.svc.cluster.local:8088 + connect: + - name: connect + url: http://connect.basic.svc.cluster.local:8083 +--- +apiVersion: platform.confluent.io/v1beta1 +kind: SchemaRegistry +metadata: + name: schemaregistry + namespace: basic +spec: + replicas: 3 + podTemplate: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - schemaregistry + topologyKey: kubernetes.io/hostname + image: + application: confluentinc/cp-schema-registry:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 +--- +apiVersion: platform.confluent.io/v1beta1 +kind: KafkaRestProxy +metadata: + name: kafkarestproxy + namespace: basic +spec: + replicas: 1 + image: + application: confluentinc/cp-kafka-rest:7.4.0 + init: confluentinc/confluent-init-container:2.6.0 + dependencies: + schemaRegistry: + url: http://schemaregistry.basic.svc.cluster.local:8081 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kafka-local +provisioner: openebs.io/local +reclaimPolicy: Retain +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/trainer-env/kafka/producer-app-data.yaml b/trainer-env/kafka/producer-app-data.yaml new file mode 100644 index 0000000..6a4d4d9 --- /dev/null +++ b/trainer-env/kafka/producer-app-data.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: kafka-client-config + namespace: basic +type: Opaque +data: + # base64 decode, then connect address + kafka.properties: Ym9vdHN0cmFwLnNlcnZlcnM9a2Fma2EuYmFzaWMuc3ZjLmNsdXN0ZXIubG9jYWw6OTA3MQpzZWN1cml0eS5wcm90b2NvbD1QTEFJTlRFWFQ= +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elastic + namespace: basic +spec: + serviceName: elastic + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + app: elastic + template: + metadata: + labels: + app: elastic + spec: + containers: + - name: elastic + image: confluentinc/cp-kafka:latest + command: + - /bin/sh + - -c + - | + kafka-producer-perf-test \ + --topic elastic-0 \ + --record-size 64 \ + --throughput 1 \ + --producer.config /mnt/kafka.properties \ + --num-records 230400 + volumeMounts: + - name: kafka-properties + mountPath: /mnt + readOnly: true + resources: + requests: + memory: 512Mi # 768Mi + cpu: 500m # 1000m + volumes: + - name: kafka-properties # Create secret with name `kafka-client-config` with client configurations + secret: + secretName: kafka-client-config +--- +apiVersion: v1 +kind: Service +metadata: + name: elastic +spec: + clusterIP: None +--- +apiVersion: platform.confluent.io/v1beta1 +kind: KafkaTopic +metadata: + name: elastic-0 + namespace: basic +spec: + replicas: 3 + # partitionCount: 3 + configs: + cleanup.policy: "delete" \ No newline at end of file