DATA-840 local kafka-connect

compose.yaml

@@ -13,30 +13,44 @@ services:
           action: sync
           target: /usr/local/app/src
 
+  sync-orgs:
+    build:
+      context: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs
+      dockerfile: Dockerfile
+    env_file:
+      - path: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs/.environment
+        required: true
+    # command: 
+      # - bin/sync_orgs.py
+      # - sleep
+      # - "900"
+    develop:
+      watch:
+        - action: rebuild
+          path: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs
+          # target: /app
+    depends_on:
+      - postgres-etl-jobs
+      - kafka
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+
   kafka:
-    image: apache/kafka-native
+    image: confluentinc/cp-kafka:6.0.1
     ports:
-      - "9092:9092"
+      - "29092:9092"
     environment:
-      # Auto create topics on first use
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-
-      # Configure listeners for both docker and host communication
-      KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093
-      KAFKA_ADVERTISED_LISTENERS: HOST://localhost:9092,DOCKER://kafka:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT
-
-      # Settings required for KRaft mode
-      KAFKA_NODE_ID: 1
-      KAFKA_PROCESS_ROLES: broker,controller
-      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091
-
-      # Listener to use for broker-to-broker communication
-      KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER
-
-      # Required for a single node cluster
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    depends_on:
+      - zookeeper
 
   kafka-ui:
     image: ghcr.io/kafbat/kafka-ui:latest
@@ -45,7 +59,7 @@ services:
     environment:
       DYNAMIC_CONFIG_ENABLED: 'true'
       KAFKA_CLUSTERS_0_NAME: local
-      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
     depends_on:
       - kafka
 
@@ -58,4 +72,71 @@ services:
     volumes:
       - ./dev/publisher:/usr/local/app
     environment:
-      KAFKA_BROKERS: kafka:9093
+      KAFKA_BROKERS: kafka:9092
+    depends_on:
+      - kafka
+
+  kafka-connect:
+    depends_on:
+    - kafka
+    image: quay.io/debezium/connect:1.9.7.Final
+    ports:
+      - "8083:8083"
+    environment:
+      GROUP_ID: "1"
+      CONFIG_STORAGE_TOPIC: "my_connect_configs"
+      OFFSET_STORAGE_TOPIC: "my_connect_offsets"
+      STATUS_STORAGE_TOPIC: "my_connect_statuses"
+      BOOTSTRAP_SERVERS: "kafka:9092"    
+
+  postgres-etl-jobs:
+    image: postgres:13.6
+    container_name: postgres-etl-jobs
+    environment:
+      POSTGRES_PASSWORD: "mysecretpassword"
+      POSTGRES_DB: "edw_pro"
+    ports:
+      - "5432:5432"
+    command: >
+      postgres -c wal_level=logical &&
+      postgres -c max_replication_slots=10 &&
+      postgres -c max_wal_senders=10
+    # volumes:
+    #   - edw.edw_pro.data:/var/lib/postgresql/data
+
+  postgres-customer-1:
+    image: postgres:13.6
+    environment:
+      POSTGRES_PASSWORD: "mysecretpassword"
+      POSTGRES_DB: "edw_pro"
+    ports:
+      - "15432:5432"
+    # volumes:
+    #   - edw.customer1.data:/var/lib/postgresql/data
+
+  plantuml:
+    image: plantuml/plantuml-server:jetty
+    ports:
+      - "18080:8080"
+
+  # furcate-task-details-topic:
+  #   build:
+  #     context: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs
+  #     dockerfile: Dockerfile
+  #   env_file:
+  #     - path: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs/.env-furcate-task-details
+  #       required: true
+  #   command: 
+  #     - bin/furcate_task_details_topic_by_org_set.py
+  #   develop:
+  #     watch:
+  #       - action: rebuild
+  #         path: ${GLYMPSE_SERVICE_TOOLS}/../../edw/etl-jobs
+  #         # target: /app
+  #   depends_on:
+  #     - postgres-etl-jobs
+  #     - kafka
+
+volumes:
+  edw.edw_pro.data:
+  edw.customer1.data: