Kafka — destinations/kafka

# examples/destinations/kafka.yaml
#
# Reference config for a Kafka destination.
#
# The publish `target` is the Kafka topic. kafka_options
# (partition_key, partitions, replication_factor) are not publish-node fields;
# messages are produced without a key and topic create uses broker defaults.

server:
  addr: ":8080"
  max_body_bytes: 1048576

destinations:
  events:
    type: kafka
    kafka:
      # One or more bootstrap brokers.
      brokers:
        - "${KAFKA_BROKER_1}"
        - "${KAFKA_BROKER_2}"
      # Producer acknowledgement level: all (default, durable) | one | none.
      acks: "all"
      # none (default) | gzip | snappy | lz4 | zstd
      compression: "snappy"
      publish_timeout: "5s"
      # SASL auth. Omit the whole block for no SASL. Mechanisms:
      # plain | scram-sha-256 | scram-sha-512.
      sasl:
        mechanism: "scram-sha-256"
        username: "${KAFKA_USERNAME}"
        password: "${KAFKA_PASSWORD}"
      # TLS. Omit or set enabled: false for plaintext. Ship the system CA
      # bundle for publicly trusted brokers; set ca_cert for private CAs.
      tls:
        enabled: true
        # ca_cert: "/etc/hookway/kafka-ca.pem"
        # client_cert: "/etc/hookway/kafka-client.crt"   # mutual TLS
        # client_key:  "/etc/hookway/kafka-client.key"   # both or neither
        # server_name: "kafka.internal"
        # insecure_skip_verify: false
      # Producer retry/backoff (maps onto the kafka-go writer).
      retry:
        max_attempts: 3
        initial_backoff: "100ms"
        max_backoff: "2s"

flows:
  webhook_events:
    nodes:
      - {id: in, type: http_trigger, path: /webhooks/events, methods: [POST]}
      - {id: body, type: read_body, max_bytes: 1048576}
      - {id: auth, type: auth, scheme: hmac_sha256_hex, secret: "${WEBHOOK_SECRET}", signature_header: X-Hub-Signature-256}
      - {id: env, type: envelope, body_format: json}
      # Kafka topic name. Previously set kafka_options.partition_key from
      # body.account.id with endpoint fallback (not a publish-node field).
      - {id: out1, type: publish, destination: events, target: billing-events, create: true}
      # No partition_key → round-robin across partitions.
      - {id: out2, type: publish, destination: events, target: audit-events}
    edges: [in>body, body>auth, auth>env, env>out1, env>out2]