Zeebe cluster setup

Hi, I am trying to set up a Zeebe cluster on three ec2 instances but getting error
zeebe_broker_2 | 2020-08-17 12:34:54.187 [] [raft-server-1-system-partition-1] WARN io.atomix.raft.roles.FollowerRole - RaftServer{system-partition-1}{role=FOLLOWER} - java.net.ConnectException

ports are open on the machine.
I tried multiple ways to setup like

  1. gateway and broker on all three instances
  2. only brokers and gateway separate
    but getting similar errors

Can someone please here help and share a sample docker-compose file to be used on three servers?

@mayank a docker compose file used on three servers? I am not aware of anything like that… you might want to look at Kubernetes for doing a multi host cluster. I wouldn’t use docker compose for that.

1 Like

Note the IP and port.
In docker-compose.yml:

services:
  zeebe:
	image: camunda/zeebe:0.23.5
	environment:
	  - ZEEiBE_LOG_LEVEL=debug
	  - ZEEBE_BROKER_NETWORK_HOST=0.0.0.0
	  - ZEEBE_BROKER_GATEWAY_CLUSTER_HOST=0.0.0.0
	  - ZEEBE_BROKER_NETWORK_COMMANDAPI_HOST=13.13.13.34        # the local host IP
	  - ZEEBE_BROKER_NETWORK_INTERNALAPI_HOST=13.13.13.34
	ports:
	  - "26500:26500"
	  - "26501:26501"
	  - "26502:26502"
	  - "9600:9600"

I’ve never done a setup like that… so I am not sure if I can help here… if someone else have done that before… please advise…

Hi ruozhixian,

Thank you for your help, I am struggling with what should be all other env variables I should mention,
is it the complete list of env variable?

Hey @mayank

you can take a look at the config in the doc. For each configuration the env variable is also specified.
https://docs.zeebe.io/appendix/broker-config-template.html#standalone-broker-with-embedded-gateway

Greets
Chris

1 Like

Hi Zelldon,

I have config like below on all three ec2 instances
version: “2”

networks:
zeebe_network:
driver: bridge

services:
node1:
container_name: zeebe_broker_2
image: camunda/zeebe:0.23.1
environment:
- ZEEBE_HOST=127.0.0.1
- ZEEBE_LOG_LEVEL=debug
- ZEEBE_BROKER_CLUSTER_NODEID=1
- ZEEBE_BROKER_CLUSTER_PARTITIONSCOUNT=2
- ZEEBE_BROKER_CLUSTER_REPLICATIONFACTOR=3
- ZEEBE_BROKER_CLUSTER_CLUSTERSIZE=3
- ZEEBE_BROKER_CLUSTER_INITIALCONTACTPOINTS=127.0.0.1:26502,node_1_ip:26502,node_2_ip:26502
ports:
- “26502:26502”
- “26501:26501”
- “26500:26500”
# volumes:
# - ./zeebe.cfg.toml:/usr/local/zeebe/conf/zeebe.cfg.toml
networks:
- zeebe_network

But I am getting the RaftServer{system-partition-1}{role=FOLLOWER} - java.net.ConnectException on all three servers

Hey @mayank

could you please post the cfg which is printed by the brokers on startup?

Greets
Chris

Hi Zelldon,

Find the config below
Broker config
{
zeebe_broker_3 | “network” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “portOffset” : 0,
zeebe_broker_3 | “maxMessageSize” : “4MB”,
zeebe_broker_3 | “advertisedHost” : “127.0.0.1”,
zeebe_broker_3 | “commandApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26501,
zeebe_broker_3 | “advertisedHost” : “127.0.0.1”,
zeebe_broker_3 | “advertisedPort” : 26501,
zeebe_broker_3 | “address” : “127.0.0.1:26501”,
zeebe_broker_3 | “advertisedAddress” : “127.0.0.1:26501”
zeebe_broker_3 | },
zeebe_broker_3 | “internalApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26502,
zeebe_broker_3 | “advertisedHost” : “127.0.0.1”,
zeebe_broker_3 | “advertisedPort” : 26502,
zeebe_broker_3 | “address” : “127.0.0.1:26502”,
zeebe_broker_3 | “advertisedAddress” : “127.0.0.1:26502”
zeebe_broker_3 | },
zeebe_broker_3 | “monitoringApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 9600,
zeebe_broker_3 | “advertisedHost” : “127.0.0.1”,
zeebe_broker_3 | “advertisedPort” : 9600,
zeebe_broker_3 | “address” : “127.0.0.1:9600”,
zeebe_broker_3 | “advertisedAddress” : “127.0.0.1:9600”
zeebe_broker_3 | },
zeebe_broker_3 | “maxMessageSizeInBytes” : 4194304
zeebe_broker_3 | },
zeebe_broker_3 | “cluster” : {
zeebe_broker_3 | “initialContactPoints” : [ “127.0.0.1:26502”, “172.31.34.170:26502”, “172.31.44.55:26502” ],
zeebe_broker_3 | “partitionIds” : [ 1, 2 ],
zeebe_broker_3 | “nodeId” : 2,
zeebe_broker_3 | “partitionsCount” : 2,
zeebe_broker_3 | “replicationFactor” : 3,
zeebe_broker_3 | “clusterSize” : 3,
zeebe_broker_3 | “clusterName” : “zeebe-cluster”,
zeebe_broker_3 | “gossipFailureTimeout” : 10000,
zeebe_broker_3 | “gossipInterval” : 250,
zeebe_broker_3 | “gossipProbeInterval” : 1000
zeebe_broker_3 | },
zeebe_broker_3 | “threads” : {
zeebe_broker_3 | “cpuThreadCount” : 2,
zeebe_broker_3 | “ioThreadCount” : 2
zeebe_broker_3 | },
zeebe_broker_3 | “data” : {
zeebe_broker_3 | “directories” : [ “/usr/local/zeebe/data” ],
zeebe_broker_3 | “logSegmentSize” : “512MB”,
zeebe_broker_3 | “snapshotPeriod” : “PT15M”,
zeebe_broker_3 | “logIndexDensity” : 100,
zeebe_broker_3 | “logSegmentSizeInBytes” : 536870912
zeebe_broker_3 | },
zeebe_broker_3 | “exporters” : { },
zeebe_broker_3 | “gateway” : {
zeebe_broker_3 | “network” : {
zeebe_broker_3 | “host” : “0.0.0.0”,
zeebe_broker_3 | “port” : 26500,
zeebe_broker_3 | “minKeepAliveInterval” : “PT30S”
zeebe_broker_3 | },
zeebe_broker_3 | “cluster” : {
zeebe_broker_3 | “contactPoint” : “127.0.0.1:26502”,
zeebe_broker_3 | “requestTimeout” : “PT15S”,
zeebe_broker_3 | “clusterName” : “zeebe-cluster”,
zeebe_broker_3 | “memberId” : “gateway”,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26502
zeebe_broker_3 | },
zeebe_broker_3 | “threads” : {
zeebe_broker_3 | “managementThreads” : 1
zeebe_broker_3 | },
zeebe_broker_3 | “monitoring” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 9600
zeebe_broker_3 | },
zeebe_broker_3 | “security” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “certificateChainPath” : null,
zeebe_broker_3 | “privateKeyPath” : null
zeebe_broker_3 | },
zeebe_broker_3 | “enable” : true
zeebe_broker_3 | },
zeebe_broker_3 | “backpressure” : {
zeebe_broker_3 | “enabled” : true,
zeebe_broker_3 | “algorithm” : “VEGAS”
zeebe_broker_3 | },
zeebe_broker_3 | “stepTimeout” : “PT5M”
zeebe_broker_3 | }

Gateway config
{
zeebe_broker_3 | “network” : {
zeebe_broker_3 | “host” : “0.0.0.0”,
zeebe_broker_3 | “port” : 26500,
zeebe_broker_3 | “minKeepAliveInterval” : “PT30S”
zeebe_broker_3 | },
zeebe_broker_3 | “cluster” : {
zeebe_broker_3 | “contactPoint” : “127.0.0.1:26502”,
zeebe_broker_3 | “requestTimeout” : “PT15S”,
zeebe_broker_3 | “clusterName” : “zeebe-cluster”,
zeebe_broker_3 | “memberId” : “gateway”,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26502
zeebe_broker_3 | },
zeebe_broker_3 | “threads” : {
zeebe_broker_3 | “managementThreads” : 1
zeebe_broker_3 | },
zeebe_broker_3 | “monitoring” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 9600
zeebe_broker_3 | },
zeebe_broker_3 | “security” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “certificateChainPath” : null,
zeebe_broker_3 | “privateKeyPath” : null
zeebe_broker_3 | },
zeebe_broker_3 | “enable” : true
zeebe_broker_3 | }

When I check the cluster status I get something like below
./zbctl --insecure status
Cluster size: 0
Partitions count: 0
Replication factor: 0
Gateway version: 0.23.1
Brokers:

I think the issue is that your advertised host is set to localhost. You need to set the host to an ip or dns name reachable by the other brokers.

You can take a look how we do it in the helm charts
https://github.com/zeebe-io/zeebe-cluster-helm/blob/master/zeebe-cluster/templates/configmap.yaml#L12

Or see a related discussion and solution

Hope that helps

Greets
Chris

Hi Zelldon,

Thanks a lot!!
The raft conn error is gone. Let me check the other things of the cluster.

2 Likes

Hi @Zelldon @salaboy

Disturbing you again, need one more help,
I am trying to start the standalone gate but it is also facing connection issue
2020-08-21 07:34:14.042 [] [atomix-0] WARN io.atomix.primitive.partition.impl.DefaultPartitionGroupMembershipService - Failed to locate management group via bootstrap nodes. Please ensure partition groups are configured either locally or remotely and the node is able to reach partition group members.
gateway | 2020-08-21 07:34:15.047 [] [atomix-partition-group-membership-service-0] WARN io.atomix.primitive.partition.impl.DefaultPartitionGroupMembershipService - Failed to locate management group via bootstrap nodes. Please ensure partition groups are configured either locally or remotely and the node is able to reach partition group members.
gateway | 2020-08-21 07:34:16.048 [] [atomix-partition-group-membership-service-0] WARN io.atomix.primitive.partition.impl.DefaultPartitionGroupMembershipService - Failed to locate management group via bootstrap nodes. Please ensure partition groups are configured either locally or remotely and the node is able to reach partition group members.

Below is config from logs
{
gateway | “network” : {
gateway | “host” : “127.0.0.0”,
gateway | “port” : 26500,
gateway | “minKeepAliveInterval” : “PT30S”
gateway | },
gateway | “cluster” : {
gateway | “contactPoint” : “172.31.34.170:26502,172.31.32.27:26502,172.31.44.55:26502”,
gateway | “requestTimeout” : “PT15S”,
gateway | “clusterName” : “zeebe-cluster”,
gateway | “memberId” : “gateway”,
gateway | “host” : “172.31.32.27”,
gateway | “port” : 26502
gateway | },
gateway | “threads” : {
gateway | “managementThreads” : 1
gateway | },
gateway | “monitoring” : {
gateway | “enabled” : false,
gateway | “host” : “0.0.0.0”,
gateway | “port” : 9600
gateway | },
gateway | “security” : {
gateway | “enabled” : false,
gateway | “certificateChainPath” : null,
gateway | “privateKeyPath” : null
gateway | }
gateway | }

any idea?

Hi, @Zelldon @salaboy @ruozhixian
Please help on the above question.

thanks,
Mayank

Hey, @mayank

for the standalone gateway contactPoint , one node is enough to be the initial contact point.

just for example, the complete docker-compose.yml:

version: "2"

services:
  gateway:
	restart: always
	container_name: gateway
	image: camunda/zeebe:0.23.4
	environment:
	  - ZEEBE_LOG_LEVEL=debug
	  - ZEEBE_STANDALONE_GATEWAY=true
	  - ZEEBE_GATEWAY_NETWORK_HOST=0.0.0.0
	  - ZEEBE_GATEWAY_NETWORK_PORT=26500
	  - ZEEBE_GATEWAY_CLUSTER_CONTACTPOINT=13.13.13.33:26502           # the initial contact node host ip
	  - ZEEBE_GATEWAY_CLUSTER_PORT=26502
	  - ZEEBE_GATEWAY_CLUSTER_HOST=13.13.13.38               # the gateway host ip
	  - ZEEBE_GATEWAY_SECURITY_ENABLED=false
	  - ZEEBE_GATEWAY_LONGPOLLING_ENABLED=true
	ports:
	  - "26500:26500"
	  - "26502:26502"

thanks @ruozhixian let me try this.

@ruozhixian @Zelldon still getting the same issue on gateway
2020-08-24 08:21:44.654 [] [atomix-0] WARN io.atomix.primitive.partition.impl.DefaultPartitionGroupMembershipService - Failed to locate management group via bootstrap nodes. Please ensure partition groups are configured either locally or remotely and the node is able to reach partition group members.
gateway | 2020-08-24 08:21:45.661 [] [atomix-partition-group-membership-service-0] WARN io.atomix.primitive.partition.impl.DefaultPartitionGroupMembershipService - Failed to locate management group via bootstrap nodes. Please ensure partition groups are configured either locally or remotely and the node is able to reach partition group members.

Gateway config is as below
{
gateway | “network” : {
gateway | “host” : “127.0.0.0”,
gateway | “port” : 26500,
gateway | “minKeepAliveInterval” : “PT30S”
gateway | },
gateway | “cluster” : {
gateway | “contactPoint” : “172.31.32.27:26502”,
gateway | “requestTimeout” : “PT15S”,
gateway | “clusterName” : “zeebe-cluster”,
gateway | “memberId” : “gateway”,
gateway | “host” : “172.31.32.27”,
gateway | “port” : 26502
gateway | },
gateway | “threads” : {
gateway | “managementThreads” : 1
gateway | },
gateway | “monitoring” : {
gateway | “enabled” : false,
gateway | “host” : “0.0.0.0”,
gateway | “port” : 9600
gateway | },
gateway | “security” : {
gateway | “enabled” : false,
gateway | “certificateChainPath” : null,
gateway | “privateKeyPath” : null
gateway | }
gateway | }

and broker config is as below
{
zeebe_broker_3 | “network” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “portOffset” : 0,
zeebe_broker_3 | “maxMessageSize” : “4MB”,
zeebe_broker_3 | “advertisedHost” : “172.31.32.27”,
zeebe_broker_3 | “commandApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26501,
zeebe_broker_3 | “advertisedHost” : “172.31.32.27”,
zeebe_broker_3 | “advertisedPort” : 26501,
zeebe_broker_3 | “address” : “127.0.0.1:26501”,
zeebe_broker_3 | “advertisedAddress” : “172.31.32.27:26501”
zeebe_broker_3 | },
zeebe_broker_3 | “internalApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26502,
zeebe_broker_3 | “advertisedHost” : “172.31.32.27”,
zeebe_broker_3 | “advertisedPort” : 26502,
zeebe_broker_3 | “address” : “127.0.0.1:26502”,
zeebe_broker_3 | “advertisedAddress” : “172.31.32.27:26502”
zeebe_broker_3 | },
zeebe_broker_3 | “monitoringApi” : {
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 9600,
zeebe_broker_3 | “advertisedHost” : “172.31.32.27”,
zeebe_broker_3 | “advertisedPort” : 9600,
zeebe_broker_3 | “address” : “127.0.0.1:9600”,
zeebe_broker_3 | “advertisedAddress” : “172.31.32.27:9600”
zeebe_broker_3 | },
zeebe_broker_3 | “maxMessageSizeInBytes” : 4194304
zeebe_broker_3 | },
zeebe_broker_3 | “cluster” : {
zeebe_broker_3 | “initialContactPoints” : [ “127.0.0.1:26502”, “172.31.34.170:26502”, “172.31.44.55:26502” ],
zeebe_broker_3 | “partitionIds” : [ 1, 2 ],
zeebe_broker_3 | “nodeId” : 2,
zeebe_broker_3 | “partitionsCount” : 2,
zeebe_broker_3 | “replicationFactor” : 3,
zeebe_broker_3 | “clusterSize” : 3,
zeebe_broker_3 | “clusterName” : “zeebe-cluster”,
zeebe_broker_3 | “gossipFailureTimeout” : 10000,
zeebe_broker_3 | “gossipInterval” : 250,
zeebe_broker_3 | “gossipProbeInterval” : 1000
zeebe_broker_3 | },
zeebe_broker_3 | “threads” : {
zeebe_broker_3 | “cpuThreadCount” : 2,
zeebe_broker_3 | “ioThreadCount” : 2
zeebe_broker_3 | },
zeebe_broker_3 | “data” : {
zeebe_broker_3 | “directories” : [ “/usr/local/zeebe/data” ],
zeebe_broker_3 | “logSegmentSize” : “512MB”,
zeebe_broker_3 | “snapshotPeriod” : “PT15M”,
zeebe_broker_3 | “logIndexDensity” : 100,
zeebe_broker_3 | “logSegmentSizeInBytes” : 536870912
zeebe_broker_3 | },
zeebe_broker_3 | “exporters” : { },
zeebe_broker_3 | “gateway” : {
zeebe_broker_3 | “network” : {
zeebe_broker_3 | “host” : “0.0.0.0”,
zeebe_broker_3 | “port” : 26500,
zeebe_broker_3 | “minKeepAliveInterval” : “PT30S”
zeebe_broker_3 | },
zeebe_broker_3 | “cluster” : {
zeebe_broker_3 | “contactPoint” : “127.0.0.1:26502”,
zeebe_broker_3 | “requestTimeout” : “PT15S”,
zeebe_broker_3 | “clusterName” : “zeebe-cluster”,
zeebe_broker_3 | “memberId” : “gateway”,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 26502
zeebe_broker_3 | },
zeebe_broker_3 | “threads” : {
zeebe_broker_3 | “managementThreads” : 1
zeebe_broker_3 | },
zeebe_broker_3 | “monitoring” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “host” : “127.0.0.1”,
zeebe_broker_3 | “port” : 9600
zeebe_broker_3 | },
zeebe_broker_3 | “security” : {
zeebe_broker_3 | “enabled” : false,
zeebe_broker_3 | “certificateChainPath” : null,
zeebe_broker_3 | “privateKeyPath” : null
zeebe_broker_3 | },
zeebe_broker_3 | “enable” : false
zeebe_broker_3 | },
zeebe_broker_3 | “backpressure” : {
zeebe_broker_3 | “enabled” : true,
zeebe_broker_3 | “algorithm” : “VEGAS”
zeebe_broker_3 | },
zeebe_broker_3 | “stepTimeout” : “PT5M”
zeebe_broker_3 | }

Hi @mayank ,

How do you resolved I am still facing the issue ?

zeebe_node1.1.nzp489et1ivk@node1 | 2020-10-09 13:21:00.361 [] [raft-server-1-system-partition-1] WARN io.atomix.raft.roles.FollowerRole - RaftServer{system-partition-1}{role=FOLLOWER} - java.net.ConnectException

Anyone can help on this?