i am facing timeout exception and network connection failed exception from zeebe and job register failed exception from java client. but it was working fine in single zeebe instance. here is my logs and config files
logs 1:
zeebe3_1 | 11:00:11.490 [] [raft-server-system-partition-1] INFO io.atomix.protocols.raft.impl.RaftContext - RaftServer{system-partition-1} - Transitioning to CANDIDATE
zeebe_1 | 11:00:06.080 [io.zeebe.gateway.impl.broker.BrokerRequestManager] [172.20.0.4:26501-zb-actors-0] WARN io.zeebe.gateway - Failed to activate jobs for type createVerificationTask from partition 2
zeebe_1 | io.zeebe.transport.RequestTimeoutException: Request timed out after PT15S
zeebe_1 | at io.zeebe.transport.impl.sender.OutgoingRequest.timeout(OutgoingRequest.java:151) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.transport.impl.sender.Sender.onTimerExpiry(Sender.java:483) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at org.agrona.DeadlineTimerWheel.poll(DeadlineTimerWheel.java:284) ~[agrona-1.0.1.jar:1.0.1]
zeebe_1 | at io.zeebe.transport.impl.sender.Sender.processTimeouts(Sender.java:114) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorJob.invoke(ActorJob.java:90) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorJob.execute(ActorJob.java:53) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorTask.execute(ActorTask.java:189) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.executeCurrentTask(ActorThread.java:154) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.doWork(ActorThread.java:135) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.run(ActorThread.java:112) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | 11:00:12.497 [exporter] [172.20.0.4:26501-zb-fs-workers-1] DEBUG io.zeebe.broker.exporter - Closed exporter director 'exporter'.
zeebe_1 | 11:00:12.434 [io.zeebe.gateway.impl.broker.BrokerRequestManager] [172.20.0.4:26501-zb-actors-0] WARN io.zeebe.gateway - Failed to activate jobs for type validateQuantity from partition 2
zeebe_1 | io.zeebe.transport.RequestTimeoutException: Request timed out after PT15S
zeebe_1 | at io.zeebe.transport.impl.sender.OutgoingRequest.timeout(OutgoingRequest.java:151) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.transport.impl.sender.Sender.onTimerExpiry(Sender.java:483) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at org.agrona.DeadlineTimerWheel.poll(DeadlineTimerWheel.java:284) ~[agrona-1.0.1.jar:1.0.1]
zeebe_1 | at io.zeebe.transport.impl.sender.Sender.processTimeouts(Sender.java:114) ~[zeebe-transport-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorJob.invoke(ActorJob.java:90) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorJob.execute(ActorJob.java:53) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorTask.execute(ActorTask.java:189) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.executeCurrentTask(ActorThread.java:154) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.doWork(ActorThread.java:135) [zeebe-util-0.18.0.jar:0.18.0]
zeebe_1 | at io.zeebe.util.sched.ActorThread.run(ActorThread.java:112) [zeebe-util-0.18.0.jar:0.18.0]
logs 2 :
zeebe2_1 | 10:59:05.642 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.net.ConnectException
zeebe_1 | 10:59:06.020 [io.zeebe.broker.clustering.base.partitions.PartitionLeaderElection] [172.20.0.4:26501-zb-actors-1] DEBUG io.zeebe.broker.clustering - Partition role transitioning from FOLLOWER to CANDIDATE
zeebe2_1 | 10:59:08.885 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe_1 | 10:59:12.871 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-1}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-poll timed out in 5000 milliseconds
zeebe2_1 | 10:59:16.097 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe2_1 | 10:59:19.027 [] [raft-server-system-partition-1] INFO io.atomix.protocols.raft.impl.RaftContext - RaftServer{system-partition-1} - Found leader 0
zeebe2_1 | 10:59:23.347 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe2_1 | 10:59:23.347 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe_1 | 10:59:22.712 [] [raft-server-raft-atomix-partition-2] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-2}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-2-vote timed out in 5000 milliseconds
zeebe3_1 | 10:59:26.330 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-1}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-poll timed out in 5000 milliseconds
zeebe2_1 | 10:59:31.211 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe2_1 | 10:59:32.691 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.CandidateRole - RaftServer{raft-atomix-partition-1}{role=CANDIDATE} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-vote timed out in 5000 milliseconds
zeebe2_1 | 10:59:32.692 [] [raft-server-system-partition-1] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{system-partition-1}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type system-partition-1-poll timed out in 5000 milliseconds
zeebe2_1 | 10:59:32.692 [] [raft-server-system-partition-1] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{system-partition-1}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type system-partition-1-poll timed out in 5000 milliseconds
zeebe2_1 | 10:59:33.725 [] [raft-server-raft-atomix-partition-2] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-2}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-2-poll timed out in 5000 milliseconds
zeebe3_1 | 10:59:32.905 [] [raft-server-system-partition-1] INFO io.atomix.protocols.raft.impl.RaftContext - RaftServer{system-partition-1} - Found leader 0
zeebe3_1 | 10:59:33.367 [] [raft-server-raft-atomix-partition-2] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-2}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-2-poll timed out in 5000 milliseconds
zeebe3_1 | 10:59:33.720 [] [raft-server-raft-atomix-partition-2] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-2}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-2-poll timed out in 5000 milliseconds
zeebe3_1 | 10:59:34.132 [] [raft-server-raft-atomix-partition-1] WARN io.atomix.protocols.raft.roles.FollowerRole - RaftServer{raft-atomix-partition-1}{role=FOLLOWER} - java.util.concurrent.TimeoutException: Request type raft-atomix-partition-1-poll timed out in 5000 milliseconds
docker-compose :
version: "2"
networks:
zeebe_network:
volumes:
zeebe_data:
zeebe2_data:
zeebe3_data:
zeebe_elasticsearch_data:
services:
zeebe:
image: camunda/zeebe:0.18.0
environment:
- ZEEBE_LOG_LEVEL=debug
- ZEEBE_NODE_ID=0
- ZEEBE_PARTITIONS_COUNT=2
- ZEEBE_REPLICATION_FACTOR=3
- ZEEBE_CLUSTER_SIZE=3
ports:
- "26500:26500"
volumes:
- zeebe_data:/usr/local/zeebe/data
- ./zeebe.cfg.toml:/usr/local/zeebe/conf/zeebe.cfg.toml
depends_on:
- elasticsearch
networks:
- zeebe_network
zeebe2:
image: camunda/zeebe:0.18.0
environment:
- ZEEBE_LOG_LEVEL=debug
- ZEEBE_NODE_ID=1
- ZEEBE_PARTITIONS_COUNT=2
- ZEEBE_REPLICATION_FACTOR=3
- ZEEBE_CLUSTER_SIZE=3
- ZEEBE_CONTACT_POINTS=zeebe:26502
volumes:
- zeebe2_data:/usr/local/zeebe/data
- ./zeebe.cfg.toml:/usr/local/zeebe/conf/zeebe.cfg.toml
depends_on:
- elasticsearch
- zeebe
networks:
- zeebe_network
zeebe3:
image: camunda/zeebe:0.18.0
environment:
- ZEEBE_LOG_LEVEL=debug
- ZEEBE_NODE_ID=2
- ZEEBE_PARTITIONS_COUNT=2
- ZEEBE_REPLICATION_FACTOR=3
- ZEEBE_CLUSTER_SIZE=3
- ZEEBE_CONTACT_POINTS=zeebe:26502
volumes:
- zeebe3_data:/usr/local/zeebe/data
- ./zeebe.cfg.toml:/usr/local/zeebe/conf/zeebe.cfg.toml
depends_on:
- elasticsearch
- zeebe2
networks:
- zeebe_network
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.7.1
ports:
- "9200:9200"
environment:
- discovery.type=single-node
- cluster.name=elasticsearch
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
volumes:
- zeebe_elasticsearch_data:/usr/share/elasticsearch/data
networks:
- zeebe_network
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.7.1
ports:
- "5601:5601"
networks:
- zeebe_network
spring boot java client code :
// register
@Configuration
public class ZeebeConfig {
@Bean
public ZeebeClient zeebe() {
return ZeebeClient.newClientBuilder().brokerContactPoint("localhost:26500").build();
}
}
// Register job
@Component("createVerificationTask-Worker")
public class CreateVerificationTaskWorker extends AbstractWorker implements JobHandler {
// Logger
private static final Logger log = LoggerFactory.getLogger(CreateVerificationTaskWorker.class);
@PostConstruct
public void subscribe() {
subscription = client.newWorker().jobType("createVerificationTask").handler(this)
.open();
}
@Override
public void handle(JobClient client, ActivatedJob job) {
componentWorkflowHandlerService.createVerificationTask(client, job);
}
@PreDestroy
public void closeSubscription() {
subscription.close();
}
}