Hello,
I’m trying to upgrade a 3 nodes zeebe cluster (partitions: 3, replication:3) from 0.24.6 to 0.25.3 but my cluster is not getting up and is stuck at the step 7/14, I’m not using a rolling update, so all my nodes a restarted when i start the upgrade. And if i check topology i get
./bin/zbctl status --insecure
Cluster size: 0
Partitions count: 0
Replication factor: 0
Gateway version: 0.25.3
Brokers:
and if i redeploy 0.24.6 version, the cluster will get up in less then 3 min.
you will find below pods logs:
2021-01-22 14:37:48.180 [] [main] INFO io.zeebe.broker.StandaloneBroker - Starting StandaloneBroker v0.25.3 on corecloud-zeebe-gra3-1 with PID 8 (/usr/local/zeebe/lib/zeebe-distribution-0.25.3.jar started by root in /usr/local/zeebe)
2021-01-22 14:37:48.192 [] [main] INFO io.zeebe.broker.StandaloneBroker - No active profile set, falling back to default profiles: default
2021-01-22 14:37:49.895 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat initialized with port(s): 9600 (http)
2021-01-22 14:37:49.909 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Initializing ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:37:49.911 [] [main] INFO org.apache.catalina.core.StandardService - Starting service [Tomcat]
2021-01-22 14:37:49.911 [] [main] INFO org.apache.catalina.core.StandardEngine - Starting Servlet engine: [Apache Tomcat/9.0.39]
2021-01-22 14:37:50.015 [] [main] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring embedded WebApplicationContext
2021-01-22 14:37:50.016 [] [main] INFO org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext - Root WebApplicationContext: initialization completed in 1772 ms
2021-01-22 14:37:50.598 [] [main] INFO org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor - Initializing ExecutorService 'applicationTaskExecutor'
2021-01-22 14:37:51.075 [] [main] INFO org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver - Exposing 4 endpoint(s) beneath base path '/actuator'
2021-01-22 14:37:51.106 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Starting ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:37:51.173 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat started on port(s): 9600 (http) with context path ''
2021-01-22 14:37:51.193 [] [main] INFO io.zeebe.broker.StandaloneBroker - Started StandaloneBroker in 3.551 seconds (JVM running for 5.066)
2021-01-22 14:37:51.254 [] [main] INFO io.zeebe.broker.system - Version: 0.25.3
2021-01-22 14:37:51.297 [] [main] INFO io.zeebe.broker.system - Starting broker 1 with configuration {
"network" : {
"host" : "0.0.0.0",
"portOffset" : 0,
"maxMessageSize" : "0MB",
"advertisedHost" : "0.0.0.0",
"commandApi" : {
"host" : "0.0.0.0",
"port" : 26501,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26501,
"advertisedAddress" : "0.0.0.0:26501",
"address" : "0.0.0.0:26501"
},
"internalApi" : {
"host" : "0.0.0.0",
"port" : 26502,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26502,
"advertisedAddress" : "0.0.0.0:26502",
"address" : "0.0.0.0:26502"
},
"monitoringApi" : {
"host" : "0.0.0.0",
"port" : 9600,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 9600,
"advertisedAddress" : "0.0.0.0:9600",
"address" : "0.0.0.0:9600"
},
"maxMessageSizeInBytes" : 131072
},
"cluster" : {
"initialContactPoints" : [ "corecloud-zeebe-gra3-0.corecloud-zeebe:26502", "corecloud-zeebe-sbg3-0.corecloud-zeebe:26502" ],
"partitionIds" : [ 1, 2, 3 ],
"nodeId" : 1,
"partitionsCount" : 3,
"replicationFactor" : 3,
"clusterSize" : 3,
"clusterName" : "zeebe-cluster",
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"cpuThreadCount" : 2,
"ioThreadCount" : 2
},
"data" : {
"directories" : [ "/usr/local/zeebe/data" ],
"logSegmentSize" : "512MB",
"snapshotPeriod" : "PT15M",
"logIndexDensity" : 100,
"diskUsageMonitoringEnabled" : true,
"diskUsageReplicationWatermark" : 0.99,
"diskUsageCommandWatermark" : 0.97,
"diskUsageMonitoringInterval" : "PT1S",
"rocksdb" : {
"columnFamilyOptions" : {
"max_write_buffer_size_to_maintain" : "1.6e+07",
"write_buffer_size" : "8e+06"
}
},
"logSegmentSizeInBytes" : 536870912,
"atomixStorageLevel" : "DISK",
"freeDiskSpaceReplicationWatermark" : 2055829463,
"freeDiskSpaceCommandWatermark" : 6167488389
},
"exporters" : {
"elasticsearch" : {
"jarPath" : null,
"className" : "io.zeebe.exporter.ElasticsearchExporter",
"args" : {
"url" : "http://corecloud-zeebe-operate-elasticsearch:9200"
},
"external" : false
}
},
"gateway" : {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
},
"backpressure" : {
"enabled" : true,
"algorithm" : "VEGAS",
"aimd" : {
"requestTimeout" : "PT1S",
"initialLimit" : 100,
"minLimit" : 1,
"maxLimit" : 1000,
"backoffRatio" : 0.9
},
"fixed" : {
"limit" : 20
},
"vegas" : {
"alpha" : 3,
"beta" : 6,
"initialLimit" : 20
},
"gradient" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0
},
"gradient2" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0,
"longWindow" : 600
}
},
"experimental" : {
"maxAppendsPerFollower" : 2,
"maxAppendBatchSize" : "0MB",
"disableExplicitRaftFlush" : false,
"detectReprocessingInconsistency" : false,
"maxAppendBatchSizeInBytes" : 32768
},
"stepTimeout" : "PT5M",
"executionMetricsExporterEnabled" : false
}
2021-01-22 14:37:51.324 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [1/14]: actor scheduler
2021-01-22 14:37:51.330 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [2/14]: membership and replication protocol
2021-01-22 14:37:51.625 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [3/14]: command api transport
2021-01-22 14:37:51.919 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [4/14]: command api handler
2021-01-22 14:37:51.963 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [5/14]: subscription api
2021-01-22 14:37:52.001 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [6/14]: embedded gateway
2021-01-22 14:37:52.017 [] [main] INFO io.zeebe.gateway - Version: 0.25.3
2021-01-22 14:37:52.018 [] [main] INFO io.zeebe.gateway - Starting gateway with configuration {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
}
2021-01-22 14:37:52.195 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-1 [7/14]: cluster services
2021-01-22 14:38:01.996 [] [raft-server-1-raft-partition-partition-2] WARN io.atomix.raft.roles.FollowerRole - RaftServer{raft-partition-partition-2}{role=FOLLOWER} - Poll request to 2 failed: java.net.ConnectException: Expected to send a message with subject 'raft-partition-partition-2-poll' to member '2', but member is not known. Known members are '[Member{id=1, address=0.0.0.0:26502, properties={event-service-topics-subscribed=Af8fAQEDAWpvYnNBdmFpbGFibOU=}}, Member{id=0, address=0.0.0.0:26502, properties={event-service-topics-subscribed=Af8fAQEDAWpvYnNBdmFpbGFibOU=}}]'.
2021-01-22 14:38:02.011 [] [raft-server-1-raft-partition-partition-2] WARN io.atomix.raft.roles.CandidateRole - RaftServer{raft-partition-partition-2}{role=CANDIDATE} - java.net.ConnectException: Expected to send a message with subject 'raft-partition-partition-2-vote' to member '2', but member is not known. Known members are '[Member{id=1, address=0.0.0.0:26502, properties={event-service-topics-subscribed=Af8fAQEDAWpvYnNBdmFpbGFibOU=}}, Member{id=0, address=0.0.0.0:26502, properties={event-service-topics-subscribed=Af8fAQEDAWpvYnNBdmFpbGFibOU=}}]'.
2021-01-22 14:38:18.559 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring DispatcherServlet 'dispatcherServlet'
2021-01-22 14:38:18.560 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Initializing Servlet 'dispatcherServlet'
2021-01-22 14:38:18.571 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Completed initialization in 11 ms
2021-01-22 14:37:58.137 [] [main] INFO io.zeebe.broker.StandaloneBroker - Starting StandaloneBroker v0.25.3 on corecloud-zeebe-sbg3-0 with PID 8 (/usr/local/zeebe/lib/zeebe-distribution-0.25.3.jar started by root in /usr/local/zeebe)
2021-01-22 14:37:58.149 [] [main] INFO io.zeebe.broker.StandaloneBroker - No active profile set, falling back to default profiles: default
2021-01-22 14:38:00.235 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat initialized with port(s): 9600 (http)
2021-01-22 14:38:00.255 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Initializing ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:38:00.257 [] [main] INFO org.apache.catalina.core.StandardService - Starting service [Tomcat]
2021-01-22 14:38:00.258 [] [main] INFO org.apache.catalina.core.StandardEngine - Starting Servlet engine: [Apache Tomcat/9.0.39]
2021-01-22 14:38:00.415 [] [main] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring embedded WebApplicationContext
2021-01-22 14:38:00.415 [] [main] INFO org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext - Root WebApplicationContext: initialization completed in 2207 ms
2021-01-22 14:38:01.014 [] [main] INFO org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor - Initializing ExecutorService 'applicationTaskExecutor'
2021-01-22 14:38:01.600 [] [main] INFO org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver - Exposing 4 endpoint(s) beneath base path '/actuator'
2021-01-22 14:38:01.633 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Starting ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:38:01.680 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat started on port(s): 9600 (http) with context path ''
2021-01-22 14:38:01.702 [] [main] INFO io.zeebe.broker.StandaloneBroker - Started StandaloneBroker in 4.181 seconds (JVM running for 5.952)
2021-01-22 14:38:01.765 [] [main] INFO io.zeebe.broker.system - Version: 0.25.3
2021-01-22 14:38:01.813 [] [main] INFO io.zeebe.broker.system - Starting broker 2 with configuration {
"network" : {
"host" : "0.0.0.0",
"portOffset" : 0,
"maxMessageSize" : "0MB",
"advertisedHost" : "0.0.0.0",
"commandApi" : {
"host" : "0.0.0.0",
"port" : 26501,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26501,
"advertisedAddress" : "0.0.0.0:26501",
"address" : "0.0.0.0:26501"
},
"internalApi" : {
"host" : "0.0.0.0",
"port" : 26502,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26502,
"advertisedAddress" : "0.0.0.0:26502",
"address" : "0.0.0.0:26502"
},
"monitoringApi" : {
"host" : "0.0.0.0",
"port" : 9600,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 9600,
"advertisedAddress" : "0.0.0.0:9600",
"address" : "0.0.0.0:9600"
},
"maxMessageSizeInBytes" : 131072
},
"cluster" : {
"initialContactPoints" : [ "corecloud-zeebe-gra3-0.corecloud-zeebe:26502", "corecloud-zeebe-gra3-1.corecloud-zeebe:26502" ],
"partitionIds" : [ 1, 2, 3 ],
"nodeId" : 2,
"partitionsCount" : 3,
"replicationFactor" : 3,
"clusterSize" : 3,
"clusterName" : "zeebe-cluster",
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"cpuThreadCount" : 2,
"ioThreadCount" : 2
},
"data" : {
"directories" : [ "/usr/local/zeebe/data" ],
"logSegmentSize" : "512MB",
"snapshotPeriod" : "PT15M",
"logIndexDensity" : 100,
"diskUsageMonitoringEnabled" : true,
"diskUsageReplicationWatermark" : 0.99,
"diskUsageCommandWatermark" : 0.97,
"diskUsageMonitoringInterval" : "PT1S",
"rocksdb" : {
"columnFamilyOptions" : {
"max_write_buffer_size_to_maintain" : "1.6e+07",
"write_buffer_size" : "8e+06"
}
},
"logSegmentSizeInBytes" : 536870912,
"atomixStorageLevel" : "DISK",
"freeDiskSpaceCommandWatermark" : 6167488389,
"freeDiskSpaceReplicationWatermark" : 2055829463
},
"exporters" : {
"elasticsearch" : {
"jarPath" : null,
"className" : "io.zeebe.exporter.ElasticsearchExporter",
"args" : {
"url" : "http://corecloud-zeebe-operate-elasticsearch:9200"
},
"external" : false
}
},
"gateway" : {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
},
"backpressure" : {
"enabled" : true,
"algorithm" : "VEGAS",
"aimd" : {
"requestTimeout" : "PT1S",
"initialLimit" : 100,
"minLimit" : 1,
"maxLimit" : 1000,
"backoffRatio" : 0.9
},
"fixed" : {
"limit" : 20
},
"vegas" : {
"alpha" : 3,
"beta" : 6,
"initialLimit" : 20
},
"gradient" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0
},
"gradient2" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0,
"longWindow" : 600
}
},
"experimental" : {
"maxAppendsPerFollower" : 2,
"maxAppendBatchSize" : "0MB",
"disableExplicitRaftFlush" : false,
"detectReprocessingInconsistency" : false,
"maxAppendBatchSizeInBytes" : 32768
},
"stepTimeout" : "PT5M",
"executionMetricsExporterEnabled" : false
}
2021-01-22 14:38:01.842 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [1/14]: actor scheduler
2021-01-22 14:38:01.849 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [2/14]: membership and replication protocol
2021-01-22 14:38:02.163 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [3/14]: command api transport
2021-01-22 14:38:02.432 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [4/14]: command api handler
2021-01-22 14:38:02.481 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [5/14]: subscription api
2021-01-22 14:38:02.523 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [6/14]: embedded gateway
2021-01-22 14:38:02.533 [] [main] INFO io.zeebe.gateway - Version: 0.25.3
2021-01-22 14:38:02.534 [] [main] INFO io.zeebe.gateway - Starting gateway with configuration {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
}
2021-01-22 14:38:02.730 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-2 [7/14]: cluster services
2021-01-22 14:38:26.901 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring DispatcherServlet 'dispatcherServlet'
2021-01-22 14:38:26.902 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Initializing Servlet 'dispatcherServlet'
2021-01-22 14:38:26.911 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Completed initialization in 8 ms
2021-01-22 14:38:44.893 [] [main] INFO io.zeebe.broker.StandaloneBroker - Starting StandaloneBroker v0.25.3 on corecloud-zeebe-gra3-0 with PID 8 (/usr/local/zeebe/lib/zeebe-distribution-0.25.3.jar started by root in /usr/local/zeebe)
2021-01-22 14:38:44.906 [] [main] INFO io.zeebe.broker.StandaloneBroker - No active profile set, falling back to default profiles: default
2021-01-22 14:38:47.237 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat initialized with port(s): 9600 (http)
2021-01-22 14:38:47.255 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Initializing ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:38:47.257 [] [main] INFO org.apache.catalina.core.StandardService - Starting service [Tomcat]
2021-01-22 14:38:47.257 [] [main] INFO org.apache.catalina.core.StandardEngine - Starting Servlet engine: [Apache Tomcat/9.0.39]
2021-01-22 14:38:47.411 [] [main] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring embedded WebApplicationContext
2021-01-22 14:38:47.412 [] [main] INFO org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext - Root WebApplicationContext: initialization completed in 2433 ms
2021-01-22 14:38:48.170 [] [main] INFO org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor - Initializing ExecutorService 'applicationTaskExecutor'
2021-01-22 14:38:48.796 [] [main] INFO org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver - Exposing 4 endpoint(s) beneath base path '/actuator'
2021-01-22 14:38:48.834 [] [main] INFO org.apache.coyote.http11.Http11NioProtocol - Starting ProtocolHandler ["http-nio-0.0.0.0-9600"]
2021-01-22 14:38:48.903 [] [main] INFO org.springframework.boot.web.embedded.tomcat.TomcatWebServer - Tomcat started on port(s): 9600 (http) with context path ''
2021-01-22 14:38:48.933 [] [main] INFO io.zeebe.broker.StandaloneBroker - Started StandaloneBroker in 4.695 seconds (JVM running for 6.645)
2021-01-22 14:38:49.022 [] [main] INFO io.zeebe.broker.system - Version: 0.25.3
2021-01-22 14:38:49.081 [] [main] INFO io.zeebe.broker.system - Starting broker 0 with configuration {
"network" : {
"host" : "0.0.0.0",
"portOffset" : 0,
"maxMessageSize" : "0MB",
"advertisedHost" : "0.0.0.0",
"commandApi" : {
"host" : "0.0.0.0",
"port" : 26501,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26501,
"advertisedAddress" : "0.0.0.0:26501",
"address" : "0.0.0.0:26501"
},
"internalApi" : {
"host" : "0.0.0.0",
"port" : 26502,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 26502,
"advertisedAddress" : "0.0.0.0:26502",
"address" : "0.0.0.0:26502"
},
"monitoringApi" : {
"host" : "0.0.0.0",
"port" : 9600,
"advertisedHost" : "0.0.0.0",
"advertisedPort" : 9600,
"advertisedAddress" : "0.0.0.0:9600",
"address" : "0.0.0.0:9600"
},
"maxMessageSizeInBytes" : 131072
},
"cluster" : {
"initialContactPoints" : [ "corecloud-zeebe-sbg3-0.corecloud-zeebe:26502", "corecloud-zeebe-gra3-1.corecloud-zeebe:26502" ],
"partitionIds" : [ 1, 2, 3 ],
"nodeId" : 0,
"partitionsCount" : 3,
"replicationFactor" : 3,
"clusterSize" : 3,
"clusterName" : "zeebe-cluster",
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"cpuThreadCount" : 2,
"ioThreadCount" : 2
},
"data" : {
"directories" : [ "/usr/local/zeebe/data" ],
"logSegmentSize" : "512MB",
"snapshotPeriod" : "PT15M",
"logIndexDensity" : 100,
"diskUsageMonitoringEnabled" : true,
"diskUsageReplicationWatermark" : 0.99,
"diskUsageCommandWatermark" : 0.97,
"diskUsageMonitoringInterval" : "PT1S",
"rocksdb" : {
"columnFamilyOptions" : {
"max_write_buffer_size_to_maintain" : "1.6e+07",
"write_buffer_size" : "8e+06"
}
},
"logSegmentSizeInBytes" : 536870912,
"atomixStorageLevel" : "DISK",
"freeDiskSpaceCommandWatermark" : 6167488389,
"freeDiskSpaceReplicationWatermark" : 2055829463
},
"exporters" : {
"elasticsearch" : {
"jarPath" : null,
"className" : "io.zeebe.exporter.ElasticsearchExporter",
"args" : {
"url" : "http://corecloud-zeebe-operate-elasticsearch:9200"
},
"external" : false
}
},
"gateway" : {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
},
"backpressure" : {
"enabled" : true,
"algorithm" : "VEGAS",
"aimd" : {
"requestTimeout" : "PT1S",
"initialLimit" : 100,
"minLimit" : 1,
"maxLimit" : 1000,
"backoffRatio" : 0.9
},
"fixed" : {
"limit" : 20
},
"vegas" : {
"alpha" : 3,
"beta" : 6,
"initialLimit" : 20
},
"gradient" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0
},
"gradient2" : {
"minLimit" : 10,
"initialLimit" : 20,
"rttTolerance" : 2.0,
"longWindow" : 600
}
},
"experimental" : {
"maxAppendsPerFollower" : 2,
"maxAppendBatchSize" : "0MB",
"disableExplicitRaftFlush" : false,
"detectReprocessingInconsistency" : false,
"maxAppendBatchSizeInBytes" : 32768
},
"stepTimeout" : "PT5M",
"executionMetricsExporterEnabled" : false
}
2021-01-22 14:38:49.118 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [1/14]: actor scheduler
2021-01-22 14:38:49.126 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [2/14]: membership and replication protocol
2021-01-22 14:38:49.459 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [3/14]: command api transport
2021-01-22 14:38:49.776 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [4/14]: command api handler
2021-01-22 14:38:49.838 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [5/14]: subscription api
2021-01-22 14:38:49.886 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [6/14]: embedded gateway
2021-01-22 14:38:49.898 [] [main] INFO io.zeebe.gateway - Version: 0.25.3
2021-01-22 14:38:49.900 [] [main] INFO io.zeebe.gateway - Starting gateway with configuration {
"network" : {
"host" : "0.0.0.0",
"port" : 26500,
"minKeepAliveInterval" : "PT30S"
},
"cluster" : {
"contactPoint" : "0.0.0.0:26502",
"requestTimeout" : "PT15S",
"clusterName" : "zeebe-cluster",
"memberId" : "gateway",
"host" : "0.0.0.0",
"port" : 26502,
"membership" : {
"broadcastUpdates" : false,
"broadcastDisputes" : true,
"notifySuspect" : false,
"gossipInterval" : "PT0.25S",
"gossipFanout" : 2,
"probeInterval" : "PT1S",
"probeTimeout" : "PT2S",
"suspectProbes" : 3,
"failureTimeout" : "PT10S",
"syncInterval" : "PT10S"
}
},
"threads" : {
"managementThreads" : 1
},
"monitoring" : {
"enabled" : true,
"host" : "0.0.0.0",
"port" : 9600
},
"security" : {
"enabled" : false,
"certificateChainPath" : null,
"privateKeyPath" : null
},
"longPolling" : {
"enabled" : true
},
"initialized" : true,
"enable" : true
}
2021-01-22 14:38:50.114 [] [main] INFO io.zeebe.broker.system - Bootstrap Broker-0 [7/14]: cluster services
2021-01-22 14:39:03.154 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring DispatcherServlet 'dispatcherServlet'
2021-01-22 14:39:03.155 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Initializing Servlet 'dispatcherServlet'
2021-01-22 14:39:03.188 [] [http-nio-0.0.0.0-9600-exec-1] INFO org.springframework.web.servlet.DispatcherServlet - Completed initialization in 33 ms