.342) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163970343, 2020-10-08 14:32:50.343) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163970342, 2020-10-08 14:32:50.342) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163970342, 2020-10-08 14:32:50.342) 2020-10-08 14:32:50,353 [tcp-comm-worker-#1] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223621104625400, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223621104625400, currTimeout=600000]], addr=/127.0.0.1:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:32:52,012 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:32:52,012 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:32:52,012 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:32:52,012 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:32:52,012 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:32:56,352 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=104s] 2020-10-08 14:32:56,352 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:32:56] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:32:56,352 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:32:56,356 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163976352, 2020-10-08 14:32:56.352) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163976352, 2020-10-08 14:32:56.352) 2020-10-08 14:32:58,013 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:32:58,013 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:32:58,013 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:32:58,013 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:32:58,013 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:00,805 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - >>> Possible starvation in striped pool. Thread name: sys-stripe-26-#27 Queue: [] Deadlock: false Completed: 8543 Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3122) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) at o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:2035) at o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) at o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) at o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) at o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at o.a.i.i.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) 2020-10-08 14:33:02,370 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=115s] 2020-10-08 14:33:02,370 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] [14:33:02] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:02,373 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:02,376 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163982374, 2020-10-08 14:33:02.374) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163982373, 2020-10-08 14:33:02.373) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163982373, 2020-10-08 14:33:02.373) 2020-10-08 14:33:04,015 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:04,015 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:04,015 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:04,015 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:04,015 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:05,376 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=113s] 2020-10-08 14:33:05,376 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:05] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:05,377 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:05,379 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163985377, 2020-10-08 14:33:05.377) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163985377, 2020-10-08 14:33:05.377) 2020-10-08 14:33:10,016 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:10,016 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:10,016 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:10,016 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:10,016 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:11,395 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=124s] 2020-10-08 14:33:11,395 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] [14:33:11] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:11,395 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:11,398 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163991395, 2020-10-08 14:33:11.395) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163991396, 2020-10-08 14:33:11.396) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163991396, 2020-10-08 14:33:11.396) 2020-10-08 14:33:14,401 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=122s] 2020-10-08 14:33:14,401 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:14] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:14,401 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:14,405 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602163994401, 2020-10-08 14:33:14.401) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602163994402, 2020-10-08 14:33:14.402) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602163994402, 2020-10-08 14:33:14.402) 2020-10-08 14:33:16,017 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:16,017 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:16,017 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:16,017 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:16,017 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:20,372 [tcp-comm-worker-#1] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223651108434000, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223651108434000, currTimeout=600000]], addr=/0:0:0:0:0:0:0:1:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:33:20,413 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=133s] 2020-10-08 14:33:20,413 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] [14:33:20] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:20,413 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:20,417 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164000413, 2020-10-08 14:33:20.413) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164000414, 2020-10-08 14:33:20.414) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164000414, 2020-10-08 14:33:20.414) 2020-10-08 14:33:22,021 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:22,021 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:22,021 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:22,021 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:22,021 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:26,429 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=134s] 2020-10-08 14:33:26,429 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:26] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:26,429 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:26,432 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=db-checkpoint-thread-#147, id=258], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164006429, 2020-10-08 14:33:26.429) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164006430, 2020-10-08 14:33:26.430) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164006430, 2020-10-08 14:33:26.430) 2020-10-08 14:33:28,022 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:28,022 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:28,022 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:28,022 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:28,022 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:30,895 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.diagnostic [] - First 10 long running transactions [total=1] 2020-10-08 14:33:30,895 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.diagnostic [] - >>> Transaction [startTime=14:31:01.822, curTime=14:33:30.798, tx=GridDhtTxLocal [nearNodeId=423a5c73-8f0f-4ffc-912f-9127a81fffe0, nearFutId=bb1f8680571-32205fe5-8ec6-4f5c-b897-c180fe1c3949, nearMiniId=1, nearFinFutId=ab1f8680571-32205fe5-8ec6-4f5c-b897-c180fe1c3949, nearFinMiniId=3, nearXidVer=GridCacheVersion [topVer=213642940, order=1602163849922, nodeOrder=13], lb=null, super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=KeySetView [], dhtNodes=KeySetView [39ecca47-3ffc-4f91-8519-3b65c1c4c09b, fbb1c088-5b9d-458a-8e87-8a6bb6a0b0a8], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=[-1243894080,-1512899836], recovery=false, mvccEnabled=false, mvccCachingCacheIds=[], txMap=ArrayList [IgniteTxEntry [txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=364, val=c72de1fd-0dbe-40fd-b6f6-be4abb6f73e4, hasValBytes=true], cacheId=-1243894080], val=[op=CREATE, val=AUDIT_VALUE_OBJECT [idHash=162686187, hash=228503311, OLDVALUE=, SUBJECTAPPLICATIONNAME=config-sync, SUBJECTADDRESS=UWSUK2W02064/10.125.215.44, TABLEKEY=PROPHETENVIRONMENTKEY [idHash=1258530003, hash=-1241127209, VALUETYPE=JStlXstPHMCMm5JA1IPl6p8Vn0YpJNzetgnd, NAME=MNNyf1m4NruVbVBzJyebO0nngehrA64bOWoM], EVTTYPE=CACHE_OBJECT_PUT, NEWVALUE=PROPHETENVIRONMENT [idHash=88334152, hash=-947305112, VERSIONKEY=CM-0050569b1dc6-1601652240784, ACTIVE=null, KEYNAME=V8H5Lr2ojmQJhBCreZsMYxVFNXSWmB5X2vgs, MODIFICATIONDATE=2020-10-08 14:31:01.823, VALUETYPE=JStlXstPHMCMm5JA1IPl6p8Vn0YpJNzetgnd, INTERNALVERSION=null, VALUE=0h3ExD5XemitU0eWoDnme3m1II0Kern6OzoH, ISACTIVE=true, NAME=MNNyf1m4NruVbVBzJyebO0nngehrA64bOWoM], SUBJECTTYPE=REMOTE_NODE, MODIFICATIONDATE=2020-10-08 14:31:01.824, SUBJECTUSERNAME=Sync User, UUID=c72de1fd-0dbe-40fd-b6f6-be4abb6f73e4, EVTMSG=sync update]], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=CacheEntryPredicate[] [], filtersPassed=false, filtersSet=false, entry=GridDhtCacheEntry [rdrs=ReaderId[] [], part=364, super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [part=364, val=c72de1fd-0dbe-40fd-b6f6-be4abb6f73e4, hasValBytes=true], val=AUDIT_VALUE_OBJECT [idHash=162686187, hash=228503311, OLDVALUE=, SUBJECTAPPLICATIONNAME=config-sync, SUBJECTADDRESS=UWSUK2W02064/10.125.215.44, TABLEKEY=PROPHETENVIRONMENTKEY [idHash=1258530003, hash=-1241127209, VALUETYPE=JStlXstPHMCMm5JA1IPl6p8Vn0YpJNzetgnd, NAME=MNNyf1m4NruVbVBzJyebO0nngehrA64bOWoM], EVTTYPE=CACHE_OBJECT_PUT, NEWVALUE=PROPHETENVIRONMENT [idHash=88334152, hash=-947305112, VERSIONKEY=CM-0050569b1dc6-1601652240784, ACTIVE=null, KEYNAME=V8H5Lr2ojmQJhBCreZsMYxVFNXSWmB5X2vgs, MODIFICATIONDATE=2020-10-08 14:31:01.823, VALUETYPE=JStlXstPHMCMm5JA1IPl6p8Vn0YpJNzetgnd, INTERNALVERSION=null, VALUE=0h3ExD5XemitU0eWoDnme3m1II0Kern6OzoH, ISACTIVE=true, NAME=MNNyf1m4NruVbVBzJyebO0nngehrA64bOWoM], SUBJECTTYPE=REMOTE_NODE, MODIFICATIONDATE=2020-10-08 14:31:01.824, SUBJECTUSERNAME=Sync User, UUID=c72de1fd-0dbe-40fd-b6f6-be4abb6f73e4, EVTMSG=sync update], ver=GridCacheVersion [topVer=213642940, order=1602163849926, nodeOrder=3], hash=-1097448454, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=LinkedList [GridCacheMvccCandidate [nodeId=43e0983b-c8f6-4921-9241-4695ffbfbd0a, ver=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3], threadId=34, id=107502, topVer=AffinityTopologyVersion [topVer=13, minorTopVer=0], reentry=null, otherNodeId=423a5c73-8f0f-4ffc-912f-9127a81fffe0, otherVer=GridCacheVersion [topVer=213642940, order=1602163849922, nodeOrder=13], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=364, val=c72de1fd-0dbe-40fd-b6f6-be4abb6f73e4, hasValBytes=true], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=2]]], prepared=0, locked=false, nodeId=null, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=143, serReadVer=null, xidVer=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3]], IgniteTxEntry [txKey=IgniteTxKey [key=PROPHETENVIRONMENTKEY [idHash=1248189797, hash=-1494800264, VALUETYPE=dashboard, NAME=TEST_694], cacheId=-1512899836], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=LinkedList [T2 {com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor@47ee834f=Object[] [PROPHETENVIRONMENT_BUILDER [idHash=676694480, hash=-260156529, VERSIONKEY=CM-0050569b1dc6-1592504475382, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=null, VALUETYPE=dashboard, INTERNALVERSION=null, VALUE=NFR, ISACTIVE=true, NAME=TEST_694], ArrayList [Column(name=MODIFICATIONDATE, type=TIMESTAMP, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=ISACTIVE, type=BOOLEAN, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=VERSIONKEY, type=VARCHAR, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=KEYNAME, type=VARCHAR, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=ACTIVE, type=BOOLEAN, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=INTERNALVERSION, type=VARCHAR, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=NAME, type=VARCHAR, schemaRef=null, isPrimaryKey=true, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=VALUE, type=VARCHAR, schemaRef=null, isPrimaryKey=false, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null), Column(name=VALUETYPE, type=VARCHAR, schemaRef=null, isPrimaryKey=true, isActive=false, isInternalVersion=false, humanReadableName=null, subColumnList=null)], PROPHETENVIRONMENT, IgniteKernal [longJVMPauseDetector=LongJVMPauseDetector [workerRef=Thread[jvm-pause-detector-worker,5,main], longPausesCnt=0, longPausesTotalDuration=0, lastWakeUpTime=1602164010796, longPausesTimestamps=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], longPausesDurations=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], cfg=IgniteConfiguration [igniteInstanceName=null, pubPoolSize=32, svcPoolSize=32, callbackPoolSize=32, stripedPoolSize=32, sysPoolSize=32, mgmtPoolSize=4, igfsPoolSize=32, dataStreamerPoolSize=32, utilityCachePoolSize=32, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=32, sqlQryHistSize=1000, dfltQryTimeout=0, igniteHome=null, igniteWorkDir=c:\data\server\single, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@11a9e7c8, nodeId=43e0983b-c8f6-4921-9241-4695ffbfbd0a, marsh=BinaryMarshaller [], marshLocJobs=false, daemon=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, sockTimeout=5000, ackTimeout=5000, marsh=JdkMarshaller [clsFilter=o.a.i.marshaller.MarshallerUtils$1@365ff890], reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=5, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=NOOP, segResolveAttempts=5, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=5000, commSpi=TcpCommunicationSpi [connectGate=o.a.i.spi.communication.tcp.TcpCommunicationSpi$ConnectGateway@69f1c9c1, connPlc=o.a.i.spi.communication.tcp.TcpCommunicationSpi$FirstConnectionPolicy@... and 2343 skipped ...MODIFICATIONDATE=2020-10-08 14:31:01.976, VALUETYPE=dashboard, INTERNALVERSION=null, VALUE=NFR, ISACTIVE=true, NAME=TEST_2582], ver=GridCacheVersion [topVer=213642940, order=1602163849926, nodeOrder=3], hash=-1189138465, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=LinkedList [GridCacheMvccCandidate [nodeId=43e0983b-c8f6-4921-9241-4695ffbfbd0a, ver=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3], threadId=34, id=107601, topVer=AffinityTopologyVersion [topVer=13, minorTopVer=0], reentry=null, otherNodeId=423a5c73-8f0f-4ffc-912f-9127a81fffe0, otherVer=GridCacheVersion [topVer=213642940, order=1602163849922, nodeOrder=13], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=PROPHETENVIRONMENTKEY [idHash=924336160, hash=-1189138465, VALUETYPE=dashboard, NAME=TEST_2582], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=2]]], prepared=0, locked=false, nodeId=null, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=153, serReadVer=null, xidVer=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3]]... and 3845 more]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3], writeVer=GridCacheVersion [topVer=213642940, order=1602163849926, nodeOrder=3], implicit=false, loc=true, threadId=34, startTime=1602163861822, nodeId=43e0983b-c8f6-4921-9241-4695ffbfbd0a, startVer=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=PESSIMISTIC, timeout=900000, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=USER_FINISH, invalidParts=null, state=COMMITTING, timedOut=false, topVer=AffinityTopologyVersion [topVer=13, minorTopVer=0], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=148976ms, onePhaseCommit=false], size=3945]]]] 2020-10-08 14:33:30,895 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.diagnostic [] - First 10 long running cache futures [total=1] 2020-10-08 14:33:30,895 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.diagnostic [] - >>> Future [startTime=14:31:06.485, curTime=14:33:30.798, fut=GridDhtTxFinishFuture [futId=b5b77680571-04146c94-ad73-45a0-a03b-ca9ddfb5481f, commit=true, xidVer=GridCacheVersion [topVer=213642940, order=1602163849925, nodeOrder=3], innerFuts=EmptyList [], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=AlwaysTrueReducer [], initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=EmptyList []]]]] 2020-10-08 14:33:30,897 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - >>> Possible starvation in striped pool. Thread name: sys-stripe-26-#27 Queue: [] Deadlock: false Completed: 8543 Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3122) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) at o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:2035) at o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) at o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) at o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) at o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at o.a.i.i.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) 2020-10-08 14:33:31,703 [wal-file-archiver%null-#115] INFO org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager [] - Starting to copy WAL segment [absIdx=7, segIdx=7, origFile=c:\data\server\single\db\wal\node00-61de2fe2-4212-495c-856f-4569ace7188e\0000000000000007.wal, dstFile=c:\data\server\single\db\wal\archive\node00-61de2fe2-4212-495c-856f-4569ace7188e\0000000000000007.wal] 2020-10-08 14:33:31,846 [wal-file-archiver%null-#115] INFO org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager [] - Copied file [src=c:\data\server\single\db\wal\node00-61de2fe2-4212-495c-856f-4569ace7188e\0000000000000007.wal, dst=c:\data\server\single\db\wal\archive\node00-61de2fe2-4212-495c-856f-4569ace7188e\0000000000000007.wal] 2020-10-08 14:33:32,433 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=145s] 2020-10-08 14:33:32,433 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] [14:33:32] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:32,433 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:32,437 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164012434, 2020-10-08 14:33:32.434) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164012434, 2020-10-08 14:33:32.434) 2020-10-08 14:33:34,024 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:34,024 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:34,024 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:34,024 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:34,024 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:35,444 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=143s] 2020-10-08 14:33:35,444 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:35] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:35,444 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:35,448 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164015444, 2020-10-08 14:33:35.444) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164015445, 2020-10-08 14:33:35.445) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164015445, 2020-10-08 14:33:35.445) 2020-10-08 14:33:40,027 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:40,027 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:40,027 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:40,027 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:40,027 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:41,457 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=154s] 2020-10-08 14:33:41,457 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=WAITING, blockCnt=0, waitCnt=7508] [14:33:41] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:41,457 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:41,460 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164021457, 2020-10-08 14:33:41.457) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164021458, 2020-10-08 14:33:41.458) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164021458, 2020-10-08 14:33:41.458) 2020-10-08 14:33:44,459 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=152s] 2020-10-08 14:33:44,459 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:44] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:44,459 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:44,462 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164024460, 2020-10-08 14:33:44.460) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164024459, 2020-10-08 14:33:44.459) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164024459, 2020-10-08 14:33:44.459) 2020-10-08 14:33:46,030 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:46,030 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:46,030 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:46,030 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:46,030 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:47,469 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=15s] 2020-10-08 14:33:47,469 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] [14:33:47] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:33:47,469 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:47,472 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-26-#27, id=57], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164027470, 2020-10-08 14:33:47.470) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164027469, 2020-10-08 14:33:47.469) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164027469, 2020-10-08 14:33:47.469) 2020-10-08 14:33:50,390 [tcp-comm-worker-#1] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223681126885900, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223681126885900, currTimeout=600000]], addr=UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:33:50,390 [tcp-comm-worker-#1] INFO org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi [] - Pinging node: 3311a67b-bfcb-41af-8c09-b2e8f2fbde9b 2020-10-08 14:33:50,390 [sys-stripe-26-#27] ERROR org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Failed to send message to remote node [node=TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true], msg=GridIoMessage [plc=2, topic=T4 [topic=TOPIC_CACHE, id1=94370aa1-e970-37ae-9471-fd583d923522, id2=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, id3=0], topicOrd=-1, ordered=true, timeout=0, skipOnTimeout=true, msg=GridContinuousMessage [type=MSG_EVT_NOTIFICATION, routineId=7dac3ff4-3460-4dc4-8324-a1ebe4561854, data=null, futId=null]]] org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3698) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.access$6800(TcpCommunicationSpi.java:287) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.processDisconnect(TcpCommunicationSpi.java:4913) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4714) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$5.body(TcpCommunicationSpi.java:2423) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:61) ~[ignite-core-2.8.1.jar:2.8.1] Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.access$6800(TcpCommunicationSpi.java:287) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.processDisconnect(TcpCommunicationSpi.java:4913) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4714) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$5.body(TcpCommunicationSpi.java:2423) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:61) ~[ignite-core-2.8.1.jar:2.8.1] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223651108434000, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 9 more Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.access$6800(TcpCommunicationSpi.java:287) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.processDisconnect(TcpCommunicationSpi.java:4913) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4714) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$5.body(TcpCommunicationSpi.java:2423) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:61) ~[ignite-core-2.8.1.jar:2.8.1] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223681126885900, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 9 more Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223621104625400, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 9 more 2020-10-08 14:33:50,395 [tcp-comm-worker-#1] INFO org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi [] - Finished node ping [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, res=false, time=12ms] 2020-10-08 14:33:50,474 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=163s] 2020-10-08 14:33:50,474 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=7509] [14:33:50] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:33:50,474 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:50,477 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164030475, 2020-10-08 14:33:50.475) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164030474, 2020-10-08 14:33:50.474) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164030474, 2020-10-08 14:33:50.474) 2020-10-08 14:33:52,031 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:52,031 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:52,031 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:52,031 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:52,031 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:33:56,482 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=164s] 2020-10-08 14:33:56,482 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:33:56] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:33:56,482 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:56,485 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164036482, 2020-10-08 14:33:56.482) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164036483, 2020-10-08 14:33:56.483) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164036483, 2020-10-08 14:33:56.483) 2020-10-08 14:33:56,485 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=24s] [14:33:56] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:33:56,485 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:33:56,486 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:33:56,500 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164036497, 2020-10-08 14:33:56.497) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164036497, 2020-10-08 14:33:56.497) 2020-10-08 14:33:58,032 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:33:58,032 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:33:58,032 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:33:58,032 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:33:58,032 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:00,900 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - >>> Possible starvation in striped pool. Thread name: sys-stripe-26-#27 Queue: [] Deadlock: false Completed: 8543 Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=7693] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7838) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3746) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) at o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:2035) at o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) at o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) at o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) at o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at o.a.i.i.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) 2020-10-08 14:34:02,525 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=176s] 2020-10-08 14:34:02,525 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=7724] [14:34:02] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:02,525 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:02,527 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164042526, 2020-10-08 14:34:02.526) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164042525, 2020-10-08 14:34:02.525) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164042525, 2020-10-08 14:34:02.525) 2020-10-08 14:34:04,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:04,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:04,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:04,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:04,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:05,878 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=174s] 2020-10-08 14:34:05,878 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:05] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:05,878 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:05,881 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164045879, 2020-10-08 14:34:05.879) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164045878, 2020-10-08 14:34:05.878) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164045879, 2020-10-08 14:34:05.879) 2020-10-08 14:34:05,882 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=34s] [14:34:05] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:05,882 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:34:05,882 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:05,896 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164045892, 2020-10-08 14:34:05.892) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-26-#27, id=57], state=RUNNABLE Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164045893, 2020-10-08 14:34:05.893) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164045893, 2020-10-08 14:34:05.893) 2020-10-08 14:34:10,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:10,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:10,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:10,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:10,034 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:11,904 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=185s] 2020-10-08 14:34:11,904 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=7906] [14:34:11] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:11,904 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:11,907 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164051904, 2020-10-08 14:34:11.904) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164051905, 2020-10-08 14:34:11.905) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164051905, 2020-10-08 14:34:11.905) 2020-10-08 14:34:16,036 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:16,036 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:16,036 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:16,036 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:16,036 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:17,912 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=186s] 2020-10-08 14:34:17,912 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:17] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:17,912 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:17,915 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164057913, 2020-10-08 14:34:17.913) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164057912, 2020-10-08 14:34:17.912) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164057912, 2020-10-08 14:34:17.912) 2020-10-08 14:34:17,915 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=46s] [14:34:17] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:17,915 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:34:17,915 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:17,927 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164057924, 2020-10-08 14:34:17.924) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164057924, 2020-10-08 14:34:17.924) 2020-10-08 14:34:20,931 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=194s] 2020-10-08 14:34:20,931 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8080] [14:34:20] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:20,931 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:20,934 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164060932, 2020-10-08 14:34:20.932) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164060931, 2020-10-08 14:34:20.931) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164060931, 2020-10-08 14:34:20.931) 2020-10-08 14:34:21,421 [sys-stripe-26-#27] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223712146125000, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223712146125000, currTimeout=600000]], addr=/127.0.0.1:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:34:22,038 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:22,038 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:22,038 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:22,038 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:22,038 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:26,940 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=195s] 2020-10-08 14:34:26,940 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:26] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:26,940 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:26,943 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164066941, 2020-10-08 14:34:26.941) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164066940, 2020-10-08 14:34:26.940) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164066940, 2020-10-08 14:34:26.940) 2020-10-08 14:34:26,943 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=55s] [14:34:26] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:26,943 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:34:26,943 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:26,955 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164066952, 2020-10-08 14:34:26.952) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164066952, 2020-10-08 14:34:26.952) 2020-10-08 14:34:28,039 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:28,039 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:28,039 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:28,039 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:28,039 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:30,903 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - >>> Possible starvation in striped pool. Thread name: sys-stripe-26-#27 Queue: [] Deadlock: false Completed: 8543 Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8273] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7838) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3746) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) at o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:2035) at o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) at o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) at o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) at o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at o.a.i.i.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) 2020-10-08 14:34:32,961 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=206s] 2020-10-08 14:34:32,961 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=RUNNABLE, blockCnt=0, waitCnt=8312] [14:34:32] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:32,961 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:32,964 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164072962, 2020-10-08 14:34:32.962) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164072961, 2020-10-08 14:34:32.961) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164072961, 2020-10-08 14:34:32.961) 2020-10-08 14:34:34,041 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:34,041 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:34,041 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:34,041 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:34,041 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:35,967 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=204s] 2020-10-08 14:34:35,967 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:35] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:35,967 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:35,970 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164075968, 2020-10-08 14:34:35.968) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164075967, 2020-10-08 14:34:35.967) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164075967, 2020-10-08 14:34:35.967) 2020-10-08 14:34:38,974 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=67s] 2020-10-08 14:34:38,974 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] [14:34:38] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:38,976 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:38,980 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164078977, 2020-10-08 14:34:38.977) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164078977, 2020-10-08 14:34:38.977) 2020-10-08 14:34:40,043 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:40,043 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:40,043 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:40,043 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:40,043 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:41,983 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=215s] 2020-10-08 14:34:41,983 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8487] [14:34:41] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:41,983 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:41,986 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164081984, 2020-10-08 14:34:41.984) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164081983, 2020-10-08 14:34:41.983) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164081983, 2020-10-08 14:34:41.983) 2020-10-08 14:34:46,045 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:46,045 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:46,045 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:46,045 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:46,045 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:48,494 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=216s] 2020-10-08 14:34:48,494 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:48] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:48,494 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:48,497 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164088495, 2020-10-08 14:34:48.495) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164088494, 2020-10-08 14:34:48.494) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164088494, 2020-10-08 14:34:48.494) 2020-10-08 14:34:48,497 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=76s] [14:34:48] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:48,497 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:34:48,498 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:48,508 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164088506, 2020-10-08 14:34:48.506) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164088505, 2020-10-08 14:34:48.505) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164088505, 2020-10-08 14:34:48.505) 2020-10-08 14:34:51,426 [sys-stripe-26-#27] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223742176046300, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223742176046300, currTimeout=600000]], addr=/0:0:0:0:0:0:0:1:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:34:52,048 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:52,048 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:52,048 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:52,048 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:52,048 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:34:54,514 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=228s] 2020-10-08 14:34:54,514 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8729] [14:34:54] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:34:54,514 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:54,517 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164094515, 2020-10-08 14:34:54.515) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164094514, 2020-10-08 14:34:54.514) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164094515, 2020-10-08 14:34:54.515) 2020-10-08 14:34:57,517 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=225s] 2020-10-08 14:34:57,517 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:34:57] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:34:57,517 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:57,520 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164097518, 2020-10-08 14:34:57.518) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164097517, 2020-10-08 14:34:57.517) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164097517, 2020-10-08 14:34:57.517) 2020-10-08 14:34:57,520 [grid-timeout-worker-#71] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=85s] [14:34:57] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:34:57,520 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:34:57,520 [grid-timeout-worker-#71] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:221) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:34:57,534 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164097531, 2020-10-08 14:34:57.531) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164097531, 2020-10-08 14:34:57.531) 2020-10-08 14:34:58,052 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:34:58,052 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:34:58,052 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:34:58,052 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:34:58,052 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:00,910 [grid-timeout-worker-#71] WARN org.apache.ignite.internal.util.typedef.G [] - >>> Possible starvation in striped pool. Thread name: sys-stripe-26-#27 Queue: [] Deadlock: false Completed: 8543 Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8853] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7838) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3746) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) at o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) at o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:2035) at o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) at o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) at o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) at o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) at o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) at o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) at o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) at o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) at o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at o.a.i.i.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) 2020-10-08 14:35:04,028 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=237s] 2020-10-08 14:35:04,028 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=8913] [14:35:04] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:35:04,028 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:04,032 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164104028, 2020-10-08 14:35:04.028) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164104029, 2020-10-08 14:35:04.029) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164104029, 2020-10-08 14:35:04.029) 2020-10-08 14:35:04,054 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:04,054 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:04,054 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:04,054 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:04,054 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:07,036 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=235s] 2020-10-08 14:35:07,036 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:35:07] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:35:07,036 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:07,039 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164107037, 2020-10-08 14:35:07.037) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164107036, 2020-10-08 14:35:07.036) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164107036, 2020-10-08 14:35:07.036) 2020-10-08 14:35:10,042 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=98s] 2020-10-08 14:35:10,042 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] [14:35:10] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:35:10,042 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:10,045 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164110043, 2020-10-08 14:35:10.043) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164110042, 2020-10-08 14:35:10.042) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164110042, 2020-10-08 14:35:10.042) 2020-10-08 14:35:10,055 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:10,055 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:10,055 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:10,055 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:10,055 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:13,454 [nio-acceptor-tcp-comm-#88] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=sys-stripe-26, threadName=sys-stripe-26-#27, blockedFor=246s] 2020-10-08 14:35:13,454 [nio-acceptor-tcp-comm-#88] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="sys-stripe-26-#27", id=57, state=TIMED_WAITING, blockCnt=0, waitCnt=9095] [14:35:13] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] 2020-10-08 14:35:13,454 [nio-acceptor-tcp-comm-#88] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485]]] org.apache.ignite.IgniteException: GridWorker [name=sys-stripe-26, igniteInstanceName=null, finished=false, heartbeatTs=1602163866485] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3024) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:13,457 [nio-acceptor-tcp-comm-#88] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164113455, 2020-10-08 14:35:13.455) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164113454, 2020-10-08 14:35:13.454) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164113454, 2020-10-08 14:35:13.454) 2020-10-08 14:35:16,059 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:16,059 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:16,059 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:16,059 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:16,059 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:19,555 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=disco-notifier-worker, threadName=disco-notifier-worker-#104, blockedFor=247s] 2020-10-08 14:35:19,555 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="disco-notifier-worker-#104", id=195, state=WAITING, blockCnt=2, waitCnt=1009] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@6ee26183, ownerName=null, ownerId=-1] [14:35:19] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] 2020-10-08 14:35:19,555 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784]]] org.apache.ignite.IgniteException: GridWorker [name=disco-notifier-worker, igniteInstanceName=null, finished=false, heartbeatTs=1602163871784] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:19,558 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164119556, 2020-10-08 14:35:19.556) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164119555, 2020-10-08 14:35:19.555) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164119555, 2020-10-08 14:35:19.555) [14:35:19] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] 2020-10-08 14:35:19,558 [ttl-cleanup-worker-#103] ERROR org.apache.ignite.internal.util.typedef.G [] - Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [workerName=db-checkpoint-thread, threadName=db-checkpoint-thread-#147, blockedFor=107s] 2020-10-08 14:35:19,558 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.util.typedef.G [] - Thread [name="db-checkpoint-thread-#147", id=258, state=WAITING, blockCnt=1, waitCnt=50] Lock [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2aa277ae, ownerName=null, ownerId=-1] 2020-10-08 14:35:19,558 [ttl-cleanup-worker-#103] WARN [] - Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=true, timeout=1000, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765]]] org.apache.ignite.IgniteException: GridWorker [name=db-checkpoint-thread, igniteInstanceName=null, finished=false, heartbeatTs=1602164011765] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1810) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3.apply(IgnitionEx.java:1805) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:234) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:194) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] 2020-10-08 14:35:19,569 [ttl-cleanup-worker-#103] WARN org.apache.ignite.internal.processors.cache.CacheDiagnosticManager [] - Page locks dump: Thread=[name=checkpoint-runner-#148, id=259], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#148 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=checkpoint-runner-#149, id=260], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#149 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=checkpoint-runner-#150, id=261], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#150 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=checkpoint-runner-#151, id=262], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#151 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#152, id=263], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#152 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#160, id=274], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#160 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#161, id=275], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#161 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#162, id=276], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#162 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#163, id=277], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#163 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#164, id=278], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#164 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#165, id=279], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#165 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#166, id=280], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#166 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#167, id=281], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#167 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#168, id=282], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#168 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#169, id=283], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#169 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#170, id=284], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#170 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#171, id=285], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#171 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#172, id=286], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#172 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#173, id=287], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#173 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#174, id=288], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#174 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#175, id=289], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#175 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#176, id=290], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#176 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#177, id=291], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#177 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#178, id=292], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#178 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#179, id=293], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#179 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#180, id=294], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#180 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=client-connector-#181, id=295], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#181 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#182, id=296], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#182 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#183, id=297], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#183 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#184, id=298], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#184 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#185, id=299], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#185 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#186, id=300], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#186 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#187, id=301], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#187 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#188, id=302], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#188 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#189, id=303], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#189 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=client-connector-#190, id=304], state=WAITING Locked pages = [] Locked pages log: name=client-connector-#190 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=db-checkpoint-thread-#147, id=258], state=WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#147 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=dms-writer-thread-#117, id=228], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#117 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=exchange-worker-#111, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#111 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-0-#1, id=31], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-1-#2, id=32], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-10-#11, id=41], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-10-#11 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-11-#12, id=42], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-11-#12 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-12-#13, id=43], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-12-#13 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-13-#14, id=44], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-13-#14 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-14-#15, id=45], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-14-#15 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-15-#16, id=46], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-15-#16 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-16-#17, id=47], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-16-#17 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-17-#18, id=48], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-17-#18 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-18-#19, id=49], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-18-#19 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-19-#20, id=50], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-19-#20 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-2-#3, id=33], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-20-#21, id=51], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-20-#21 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-21-#22, id=52], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-21-#22 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-22-#23, id=53], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-22-#23 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-23-#24, id=54], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-23-#24 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-24-#25, id=55], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-24-#25 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-25-#26, id=56], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-25-#26 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-26-#27, id=57], state=TIMED_WAITING Locked pages = [] Locked pages log: name=sys-stripe-26-#27 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-27-#28, id=58], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-27-#28 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-28-#29, id=59], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-28-#29 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-29-#30, id=60], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-29-#30 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-3-#4, id=34], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-30-#31, id=61], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-30-#31 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-31-#32, id=62], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-31-#32 time=(1602164119567, 2020-10-08 14:35:19.567) Thread=[name=sys-stripe-4-#5, id=35], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-5-#6, id=36], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-6-#7, id=37], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-7-#8, id=38], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-8-#9, id=39], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-8-#9 time=(1602164119566, 2020-10-08 14:35:19.566) Thread=[name=sys-stripe-9-#10, id=40], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-9-#10 time=(1602164119567, 2020-10-08 14:35:19.567) 2020-10-08 14:35:21,465 [sys-stripe-26-#27] WARN org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Handshake timed out (will stop attempts to perform the handshake) [node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, connTimeoutStrategy=ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223772180706400, currTimeout=600000], err=Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223772180706400, currTimeout=600000]], addr=UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103, failureDetectionTimeoutEnabled=true, timeout=0] 2020-10-08 14:35:21,465 [sys-stripe-26-#27] ERROR org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Failed to send message to remote node [node=TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true], msg=GridIoMessage [plc=2, topic=T4 [topic=TOPIC_CACHE, id1=94370aa1-e970-37ae-9471-fd583d923522, id2=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, id3=0], topicOrd=-1, ordered=true, timeout=0, skipOnTimeout=true, msg=GridContinuousMessage [type=MSG_EVT_NOTIFICATION, routineId=7dac3ff4-3460-4dc4-8324-a1ebe4561854, data=null, futId=null]]] org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3698) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223742176046300, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 41 more Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223772180706400, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 41 more Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223712146125000, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 41 more 2020-10-08 14:35:21,467 [sys-stripe-26-#27] ERROR org.apache.ignite.continuous.query [] - Failed to send event notification to node: 3311a67b-bfcb-41af-8c09-b2e8f2fbde9b org.apache.ignite.IgniteCheckedException: Failed to send message (node may have left the grid or TCP connection cannot be established due to firewall issues) [node=TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true], topic=T4 [topic=TOPIC_CACHE, id1=94370aa1-e970-37ae-9471-fd583d923522, id2=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, id3=0], msg=GridContinuousMessage [type=MSG_EVT_NOTIFICATION, routineId=7dac3ff4-3460-4dc4-8324-a1ebe4561854, data=null, futId=null], policy=2] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2046) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Caused by: org.apache.ignite.spi.IgniteSpiException: Failed to send message to remote node: TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2955) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] ... 35 more Caused by: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3698) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] ... 35 more Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223742176046300, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 41 more Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=[/127.0.0.1:47103, /0:0:0:0:0:0:0:1:47103, UWSUK2W02064.CMC.LOCAL/10.125.215.44:47103]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3700) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:2280) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1963) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1934) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1916) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1323) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1260) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:1045) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:90) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:445) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:443) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1588) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:745) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:606) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:478) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:523) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1077) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:953) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:909) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:123) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:217) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:215) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) ~[ignite-core-2.8.1.jar:2.8.1] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181] Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223772180706400, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] ... 41 more Caused by: org.apache.ignite.spi.IgniteSpiOperationTimeoutException: Operation timed out [timeoutStrategy= ExponentialBackoffTimeoutStrategy [maxTimeout=600000, totalTimeout=30000, startNanos=223712146125000, currTimeout=600000]] at org.apache.ignite.spi.ExponentialBackoffTimeoutStrategy.nextTimeout(ExponentialBackoffTimeoutStrategy.java:103) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.TimeoutStrategy.nextTimeout(TimeoutStrategy.java:39) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioSession(TcpCommunicationSpi.java:3582) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3458) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createCommunicationClient(TcpCommunicationSpi.java:3198) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:3078) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2918) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2877) ~[ignite-core-2.8.1.jar:2.8.1] at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:2035) ~[ignite-core-2.8.1.jar:2.8.1] ... 35 more 2020-10-08 14:35:21,472 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=942118450, hash=-1795064201, VALUETYPE=LivePrices, NAME=TEST144], Value:PROPHETENVIRONMENT [idHash=1353950092, hash=-1946898991, VERSIONKEY=CM-0050569b1dc6-1598544781482, ACTIVE=null, KEYNAME=TEST145, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=TEST144], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,472 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,472 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=942118450, hash=-1795064201, VALUETYPE=LivePrices, NAME=TEST144], newVal=PROPHETENVIRONMENT [idHash=1353950092, hash=-1946898991, VERSIONKEY=CM-0050569b1dc6-1598544781482, ACTIVE=null, KEYNAME=TEST145, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=TEST144], oldVal=null, partCntr=111] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,472 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=942118450, hash=-1795064201, VALUETYPE=LivePrices, NAME=TEST144], Value:PROPHETENVIRONMENT [idHash=1353950092, hash=-1946898991, VERSIONKEY=CM-0050569b1dc6-1598544781482, ACTIVE=null, KEYNAME=TEST145, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=TEST144], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,472 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,472 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=942118450, hash=-1795064201, VALUETYPE=LivePrices, NAME=TEST144], newVal=PROPHETENVIRONMENT [idHash=1353950092, hash=-1946898991, VERSIONKEY=CM-0050569b1dc6-1598544781482, ACTIVE=null, KEYNAME=TEST145, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=TEST144], oldVal=null, partCntr=111] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,481 [grid-nio-worker-tcp-comm-7-#79] INFO org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi [] - Established outgoing communication connection [locAddr=/127.0.0.1:50342, rmtAddr=/127.0.0.1:47104] 2020-10-08 14:35:21,482 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=407245009, hash=171836934, VALUETYPE=LivePrices, NAME=1SHAM_704], Value:PROPHETENVIRONMENT [idHash=1170117199, hash=86322520, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_704], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=407245009, hash=171836934, VALUETYPE=LivePrices, NAME=1SHAM_704], newVal=PROPHETENVIRONMENT [idHash=1170117199, hash=86322520, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_704], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=407245009, hash=171836934, VALUETYPE=LivePrices, NAME=1SHAM_704], Value:PROPHETENVIRONMENT [idHash=1170117199, hash=86322520, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_704], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=407245009, hash=171836934, VALUETYPE=LivePrices, NAME=1SHAM_704], newVal=PROPHETENVIRONMENT [idHash=1170117199, hash=86322520, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.186, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_704], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=2055243028, hash=-1359765597, VALUETYPE=LivePrices, NAME=1SHAM_707], Value:PROPHETENVIRONMENT [idHash=316387633, hash=-1037642535, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.189, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_707], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=2055243028, hash=-1359765597, VALUETYPE=LivePrices, NAME=1SHAM_707], newVal=PROPHETENVIRONMENT [idHash=316387633, hash=-1037642535, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.189, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_707], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=2055243028, hash=-1359765597, VALUETYPE=LivePrices, NAME=1SHAM_707], Value:PROPHETENVIRONMENT [idHash=316387633, hash=-1037642535, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.189, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_707], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,483 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=2055243028, hash=-1359765597, VALUETYPE=LivePrices, NAME=1SHAM_707], newVal=PROPHETENVIRONMENT [idHash=316387633, hash=-1037642535, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.189, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_707], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1223197889, hash=-849231420, VALUETYPE=LivePrices, NAME=1SHAM_706], Value:PROPHETENVIRONMENT [idHash=970232379, hash=180488218, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.19, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_706], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1223197889, hash=-849231420, VALUETYPE=LivePrices, NAME=1SHAM_706], newVal=PROPHETENVIRONMENT [idHash=970232379, hash=180488218, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.19, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_706], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1223197889, hash=-849231420, VALUETYPE=LivePrices, NAME=1SHAM_706], Value:PROPHETENVIRONMENT [idHash=970232379, hash=180488218, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.19, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_706], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1223197889, hash=-849231420, VALUETYPE=LivePrices, NAME=1SHAM_706], newVal=PROPHETENVIRONMENT [idHash=970232379, hash=180488218, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.19, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_706], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=552166165, hash=1014612454, VALUETYPE=LivePrices, NAME=1SHAM_715], Value:PROPHETENVIRONMENT [idHash=1742076039, hash=-800614622, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.191, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_715], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=552166165, hash=1014612454, VALUETYPE=LivePrices, NAME=1SHAM_715], newVal=PROPHETENVIRONMENT [idHash=1742076039, hash=-800614622, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.191, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_715], oldVal=null, partCntr=245] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=552166165, hash=1014612454, VALUETYPE=LivePrices, NAME=1SHAM_715], Value:PROPHETENVIRONMENT [idHash=1742076039, hash=-800614622, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.191, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_715], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,484 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=552166165, hash=1014612454, VALUETYPE=LivePrices, NAME=1SHAM_715], newVal=PROPHETENVIRONMENT [idHash=1742076039, hash=-800614622, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.191, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_715], oldVal=null, partCntr=245] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1690644606, hash=1525146631, VALUETYPE=LivePrices, NAME=1SHAM_714], Value:PROPHETENVIRONMENT [idHash=950695752, hash=417516131, VERSIONKEY=CM-0050569b1dc6-1592504475674, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.192, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_714], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1690644606, hash=1525146631, VALUETYPE=LivePrices, NAME=1SHAM_714], newVal=PROPHETENVIRONMENT [idHash=950695752, hash=417516131, VERSIONKEY=CM-0050569b1dc6-1592504475674, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.192, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_714], oldVal=null, partCntr=161] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1690644606, hash=1525146631, VALUETYPE=LivePrices, NAME=1SHAM_714], Value:PROPHETENVIRONMENT [idHash=950695752, hash=417516131, VERSIONKEY=CM-0050569b1dc6-1592504475674, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.192, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_714], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1690644606, hash=1525146631, VALUETYPE=LivePrices, NAME=1SHAM_714], newVal=PROPHETENVIRONMENT [idHash=950695752, hash=417516131, VERSIONKEY=CM-0050569b1dc6-1592504475674, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.192, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_714], oldVal=null, partCntr=161] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=763289237, hash=-1870299774, VALUETYPE=LivePrices, NAME=1SHAM_708], Value:PROPHETENVIRONMENT [idHash=484667693, hash=-876470084, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_708], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=763289237, hash=-1870299774, VALUETYPE=LivePrices, NAME=1SHAM_708], newVal=PROPHETENVIRONMENT [idHash=484667693, hash=-876470084, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_708], oldVal=null, partCntr=147] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=763289237, hash=-1870299774, VALUETYPE=LivePrices, NAME=1SHAM_708], Value:PROPHETENVIRONMENT [idHash=484667693, hash=-876470084, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_708], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,485 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=763289237, hash=-1870299774, VALUETYPE=LivePrices, NAME=1SHAM_708], newVal=PROPHETENVIRONMENT [idHash=484667693, hash=-876470084, VERSIONKEY=CM-0050569b1dc6-1592504475677, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_708], oldVal=null, partCntr=147] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=126662619, hash=-338697243, VALUETYPE=LivePrices, NAME=1SHAM_705], Value:PROPHETENVIRONMENT [idHash=1094193733, hash=-187352825, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_705], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=126662619, hash=-338697243, VALUETYPE=LivePrices, NAME=1SHAM_705], newVal=PROPHETENVIRONMENT [idHash=1094193733, hash=-187352825, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_705], oldVal=null, partCntr=154] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=126662619, hash=-338697243, VALUETYPE=LivePrices, NAME=1SHAM_705], Value:PROPHETENVIRONMENT [idHash=1094193733, hash=-187352825, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_705], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=126662619, hash=-338697243, VALUETYPE=LivePrices, NAME=1SHAM_705], newVal=PROPHETENVIRONMENT [idHash=1094193733, hash=-187352825, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.193, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_705], oldVal=null, partCntr=154] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1192050713, hash=1703439465, VALUETYPE=LivePrices, NAME=1SHAM_701], Value:PROPHETENVIRONMENT [idHash=1203330803, hash=1211460812, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.194, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_701], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1192050713, hash=1703439465, VALUETYPE=LivePrices, NAME=1SHAM_701], newVal=PROPHETENVIRONMENT [idHash=1203330803, hash=1211460812, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.194, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_701], oldVal=null, partCntr=119] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1192050713, hash=1703439465, VALUETYPE=LivePrices, NAME=1SHAM_701], Value:PROPHETENVIRONMENT [idHash=1203330803, hash=1211460812, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.194, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_701], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,486 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1192050713, hash=1703439465, VALUETYPE=LivePrices, NAME=1SHAM_701], newVal=PROPHETENVIRONMENT [idHash=1203330803, hash=1211460812, VERSIONKEY=CM-0050569b1dc6-1592504475680, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.194, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_701], oldVal=null, partCntr=119] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=840279689, hash=47152750, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], Value:PROPHETENVIRONMENT [idHash=1969334304, hash=836470910, VERSIONKEY=CM-0050569b1dc6-1592546326355, ACTIVE=null, KEYNAME=5YH0wTPZmilUX2pfCqou3jY9IbqsRy64ATVE, MODIFICATIONDATE=2020-10-08 14:31:06.195, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, INTERNALVERSION=null, VALUE=Yp2s2J7jIThcWUkblrd4nmOORlOKzkTWpXxF, ISACTIVE=true, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=840279689, hash=47152750, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], newVal=PROPHETENVIRONMENT [idHash=1969334304, hash=836470910, VERSIONKEY=CM-0050569b1dc6-1592546326355, ACTIVE=null, KEYNAME=5YH0wTPZmilUX2pfCqou3jY9IbqsRy64ATVE, MODIFICATIONDATE=2020-10-08 14:31:06.195, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, INTERNALVERSION=null, VALUE=Yp2s2J7jIThcWUkblrd4nmOORlOKzkTWpXxF, ISACTIVE=true, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=840279689, hash=47152750, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], Value:PROPHETENVIRONMENT [idHash=1969334304, hash=836470910, VERSIONKEY=CM-0050569b1dc6-1592546326355, ACTIVE=null, KEYNAME=5YH0wTPZmilUX2pfCqou3jY9IbqsRy64ATVE, MODIFICATIONDATE=2020-10-08 14:31:06.195, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, INTERNALVERSION=null, VALUE=Yp2s2J7jIThcWUkblrd4nmOORlOKzkTWpXxF, ISACTIVE=true, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=840279689, hash=47152750, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], newVal=PROPHETENVIRONMENT [idHash=1969334304, hash=836470910, VERSIONKEY=CM-0050569b1dc6-1592546326355, ACTIVE=null, KEYNAME=5YH0wTPZmilUX2pfCqou3jY9IbqsRy64ATVE, MODIFICATIONDATE=2020-10-08 14:31:06.195, VALUETYPE=X3ktrjTl0Z6oQAu93pnKodBdDQhEV6ZhNuiL, INTERNALVERSION=null, VALUE=Yp2s2J7jIThcWUkblrd4nmOORlOKzkTWpXxF, ISACTIVE=true, NAME=CyYaGD6gTnMfCj8s1B069howOjXwybrvzWVt], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=239375636, hash=1914133345, VALUETYPE=LivePrices, NAME=1SHAM_709], Value:PROPHETENVIRONMENT [idHash=431557542, hash=-694960258, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.197, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_709], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=239375636, hash=1914133345, VALUETYPE=LivePrices, NAME=1SHAM_709], newVal=PROPHETENVIRONMENT [idHash=431557542, hash=-694960258, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.197, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_709], oldVal=null, partCntr=112] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=239375636, hash=1914133345, VALUETYPE=LivePrices, NAME=1SHAM_709], Value:PROPHETENVIRONMENT [idHash=431557542, hash=-694960258, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.197, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_709], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,487 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=239375636, hash=1914133345, VALUETYPE=LivePrices, NAME=1SHAM_709], newVal=PROPHETENVIRONMENT [idHash=431557542, hash=-694960258, VERSIONKEY=CM-0050569b1dc6-1592504475678, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.197, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_709], oldVal=null, partCntr=112] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=94490976, hash=-852058642, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], Value:PROPHETENVIRONMENT [idHash=1144552917, hash=482530371, VERSIONKEY=CM-0050569b1dc6-1601903600438, ACTIVE=null, KEYNAME=F52X8Kv8gZPuJ59wGxDx86qLrra07bJCGKRU, MODIFICATIONDATE=2020-10-08 14:31:06.204, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, INTERNALVERSION=null, VALUE=4LgvTLOdi5dHljEOL6isHej0XcKQh5FCCrgt, ISACTIVE=true, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=94490976, hash=-852058642, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], newVal=PROPHETENVIRONMENT [idHash=1144552917, hash=482530371, VERSIONKEY=CM-0050569b1dc6-1601903600438, ACTIVE=null, KEYNAME=F52X8Kv8gZPuJ59wGxDx86qLrra07bJCGKRU, MODIFICATIONDATE=2020-10-08 14:31:06.204, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, INTERNALVERSION=null, VALUE=4LgvTLOdi5dHljEOL6isHej0XcKQh5FCCrgt, ISACTIVE=true, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=94490976, hash=-852058642, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], Value:PROPHETENVIRONMENT [idHash=1144552917, hash=482530371, VERSIONKEY=CM-0050569b1dc6-1601903600438, ACTIVE=null, KEYNAME=F52X8Kv8gZPuJ59wGxDx86qLrra07bJCGKRU, MODIFICATIONDATE=2020-10-08 14:31:06.204, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, INTERNALVERSION=null, VALUE=4LgvTLOdi5dHljEOL6isHej0XcKQh5FCCrgt, ISACTIVE=true, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=94490976, hash=-852058642, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], newVal=PROPHETENVIRONMENT [idHash=1144552917, hash=482530371, VERSIONKEY=CM-0050569b1dc6-1601903600438, ACTIVE=null, KEYNAME=F52X8Kv8gZPuJ59wGxDx86qLrra07bJCGKRU, MODIFICATIONDATE=2020-10-08 14:31:06.204, VALUETYPE=qmF232x1Yv69ft4D1XzYOiVGtjAadoOk5yGU, INTERNALVERSION=null, VALUE=4LgvTLOdi5dHljEOL6isHej0XcKQh5FCCrgt, ISACTIVE=true, NAME=ISPAF27hhVZ7qxOFS1FnDQ9LVZVC3hAcR488], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,488 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=140047281, hash=645484790, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], Value:PROPHETENVIRONMENT [idHash=1438591411, hash=1081825848, VERSIONKEY=CM-0050569b1dc6-1598616441714, ACTIVE=null, KEYNAME=SNl3m0Vi7QEfiXowLZ8JXtqJj1qcE6dB1MkU, MODIFICATIONDATE=2020-10-08 14:31:06.207, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, INTERNALVERSION=null, VALUE=Ku2XMUpjXJF4GcKCX1B9uSQuDwEzUsYRWov3, ISACTIVE=true, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=140047281, hash=645484790, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], newVal=PROPHETENVIRONMENT [idHash=1438591411, hash=1081825848, VERSIONKEY=CM-0050569b1dc6-1598616441714, ACTIVE=null, KEYNAME=SNl3m0Vi7QEfiXowLZ8JXtqJj1qcE6dB1MkU, MODIFICATIONDATE=2020-10-08 14:31:06.207, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, INTERNALVERSION=null, VALUE=Ku2XMUpjXJF4GcKCX1B9uSQuDwEzUsYRWov3, ISACTIVE=true, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null [14:35:21] Topology snapshot [ver=15, locNode=43e0983b, servers=3, clients=2, state=ACTIVE, CPUs=32, offheap=12.0GB, heap=35.0GB] 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=140047281, hash=645484790, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], Value:PROPHETENVIRONMENT [idHash=1438591411, hash=1081825848, VERSIONKEY=CM-0050569b1dc6-1598616441714, ACTIVE=null, KEYNAME=SNl3m0Vi7QEfiXowLZ8JXtqJj1qcE6dB1MkU, MODIFICATIONDATE=2020-10-08 14:31:06.207, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, INTERNALVERSION=null, VALUE=Ku2XMUpjXJF4GcKCX1B9uSQuDwEzUsYRWov3, ISACTIVE=true, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl,Service:null,Instance:pricingClient, HostName:null [14:35:21] ^-- Baseline [id=0, size=3, online=3, offline=0] 2020-10-08 14:35:21,489 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=140047281, hash=645484790, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], newVal=PROPHETENVIRONMENT [idHash=1438591411, hash=1081825848, VERSIONKEY=CM-0050569b1dc6-1598616441714, ACTIVE=null, KEYNAME=SNl3m0Vi7QEfiXowLZ8JXtqJj1qcE6dB1MkU, MODIFICATIONDATE=2020-10-08 14:31:06.207, VALUETYPE=LEVoK0bmJzx4QaiXyhVdKbPluXZOnnpFlYLl, INTERNALVERSION=null, VALUE=Ku2XMUpjXJF4GcKCX1B9uSQuDwEzUsYRWov3, ISACTIVE=true, NAME=N2QK5db3WX0gFIV2ifiQkRbdS7FJ7ibgGSUJ], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=989477589, hash=-1927045145, VALUETYPE=LivePrices, NAME=1SHAM_725], Value:PROPHETENVIRONMENT [idHash=1763942753, hash=-1757265320, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.211, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_725], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=989477589, hash=-1927045145, VALUETYPE=LivePrices, NAME=1SHAM_725], newVal=PROPHETENVIRONMENT [idHash=1763942753, hash=-1757265320, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.211, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_725], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=989477589, hash=-1927045145, VALUETYPE=LivePrices, NAME=1SHAM_725], Value:PROPHETENVIRONMENT [idHash=1763942753, hash=-1757265320, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.211, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_725], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,489 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=989477589, hash=-1927045145, VALUETYPE=LivePrices, NAME=1SHAM_725], newVal=PROPHETENVIRONMENT [idHash=1763942753, hash=-1757265320, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.211, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_725], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1589832428, hash=-395442614, VALUETYPE=LivePrices, NAME=1SHAM_722], Value:PROPHETENVIRONMENT [idHash=378657087, hash=-425458436, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.212, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_722], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1589832428, hash=-395442614, VALUETYPE=LivePrices, NAME=1SHAM_722], newVal=PROPHETENVIRONMENT [idHash=378657087, hash=-425458436, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.212, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_722], oldVal=null, partCntr=132] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1589832428, hash=-395442614, VALUETYPE=LivePrices, NAME=1SHAM_722], Value:PROPHETENVIRONMENT [idHash=378657087, hash=-425458436, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.212, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_722], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1589832428, hash=-395442614, VALUETYPE=LivePrices, NAME=1SHAM_722], newVal=PROPHETENVIRONMENT [idHash=378657087, hash=-425458436, VERSIONKEY=CM-0050569b1dc6-1592504475672, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.212, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_722], oldVal=null, partCntr=132] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=931736998, hash=115091563, VALUETYPE=LivePrices, NAME=1SHAM_721], Value:PROPHETENVIRONMENT [idHash=431595505, hash=792672317, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.213, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_721], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=931736998, hash=115091563, VALUETYPE=LivePrices, NAME=1SHAM_721], newVal=PROPHETENVIRONMENT [idHash=431595505, hash=792672317, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.213, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_721], oldVal=null, partCntr=182] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=931736998, hash=115091563, VALUETYPE=LivePrices, NAME=1SHAM_721], Value:PROPHETENVIRONMENT [idHash=431595505, hash=792672317, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.213, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_721], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,490 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=931736998, hash=115091563, VALUETYPE=LivePrices, NAME=1SHAM_721], newVal=PROPHETENVIRONMENT [idHash=431595505, hash=792672317, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.213, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_721], oldVal=null, partCntr=182] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=935025174, hash=811421872, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], Value:PROPHETENVIRONMENT [idHash=1193015274, hash=-654949426, VERSIONKEY=CM-0050569b1dc6-1592546326513, ACTIVE=null, KEYNAME=XNY9FX9NASQ028pu3IxAkbZsPnr893GZtJgd, MODIFICATIONDATE=2020-10-08 14:31:06.216, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, INTERNALVERSION=null, VALUE=2XS1BkGGVrEK1PjBojy2VSpDZEa4qgl0nnxB, ISACTIVE=true, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=935025174, hash=811421872, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], newVal=PROPHETENVIRONMENT [idHash=1193015274, hash=-654949426, VERSIONKEY=CM-0050569b1dc6-1592546326513, ACTIVE=null, KEYNAME=XNY9FX9NASQ028pu3IxAkbZsPnr893GZtJgd, MODIFICATIONDATE=2020-10-08 14:31:06.216, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, INTERNALVERSION=null, VALUE=2XS1BkGGVrEK1PjBojy2VSpDZEa4qgl0nnxB, ISACTIVE=true, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], oldVal=null, partCntr=210] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=935025174, hash=811421872, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], Value:PROPHETENVIRONMENT [idHash=1193015274, hash=-654949426, VERSIONKEY=CM-0050569b1dc6-1592546326513, ACTIVE=null, KEYNAME=XNY9FX9NASQ028pu3IxAkbZsPnr893GZtJgd, MODIFICATIONDATE=2020-10-08 14:31:06.216, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, INTERNALVERSION=null, VALUE=2XS1BkGGVrEK1PjBojy2VSpDZEa4qgl0nnxB, ISACTIVE=true, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=935025174, hash=811421872, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], newVal=PROPHETENVIRONMENT [idHash=1193015274, hash=-654949426, VERSIONKEY=CM-0050569b1dc6-1592546326513, ACTIVE=null, KEYNAME=XNY9FX9NASQ028pu3IxAkbZsPnr893GZtJgd, MODIFICATIONDATE=2020-10-08 14:31:06.216, VALUETYPE=gd0UAz9HWXQ06y9Ymumn3flIUupJYHjbBrJK, INTERNALVERSION=null, VALUE=2XS1BkGGVrEK1PjBojy2VSpDZEa4qgl0nnxB, ISACTIVE=true, NAME=ZFlgfTiYsDTFkyjPlKxqe5r5pADjLhkK6BLO], oldVal=null, partCntr=210] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1647512367, hash=-905976791, VALUETYPE=LivePrices, NAME=1SHAM_723], Value:PROPHETENVIRONMENT [idHash=1463053445, hash=886838015, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.217, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_723], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1647512367, hash=-905976791, VALUETYPE=LivePrices, NAME=1SHAM_723], newVal=PROPHETENVIRONMENT [idHash=1463053445, hash=886838015, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.217, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_723], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1647512367, hash=-905976791, VALUETYPE=LivePrices, NAME=1SHAM_723], Value:PROPHETENVIRONMENT [idHash=1463053445, hash=886838015, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.217, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_723], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,491 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1647512367, hash=-905976791, VALUETYPE=LivePrices, NAME=1SHAM_723], newVal=PROPHETENVIRONMENT [idHash=1463053445, hash=886838015, VERSIONKEY=CM-0050569b1dc6-1592504475673, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.217, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_723], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=2069918539, hash=-1494807473, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], Value:PROPHETENVIRONMENT [idHash=1282994500, hash=701842127, VERSIONKEY=CM-0050569b1dc6-1592546326608, ACTIVE=null, KEYNAME=rVNTyk8hQI82wMmHtWqUSk7kzGRaWCthbhA0, MODIFICATIONDATE=2020-10-08 14:31:06.23, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, INTERNALVERSION=null, VALUE=Hfhi2SbCcv7AHUu7atbWNr67wyh6ykTvrScy, ISACTIVE=true, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=2069918539, hash=-1494807473, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], newVal=PROPHETENVIRONMENT [idHash=1282994500, hash=701842127, VERSIONKEY=CM-0050569b1dc6-1592546326608, ACTIVE=null, KEYNAME=rVNTyk8hQI82wMmHtWqUSk7kzGRaWCthbhA0, MODIFICATIONDATE=2020-10-08 14:31:06.23, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, INTERNALVERSION=null, VALUE=Hfhi2SbCcv7AHUu7atbWNr67wyh6ykTvrScy, ISACTIVE=true, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=2069918539, hash=-1494807473, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], Value:PROPHETENVIRONMENT [idHash=1282994500, hash=701842127, VERSIONKEY=CM-0050569b1dc6-1592546326608, ACTIVE=null, KEYNAME=rVNTyk8hQI82wMmHtWqUSk7kzGRaWCthbhA0, MODIFICATIONDATE=2020-10-08 14:31:06.23, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, INTERNALVERSION=null, VALUE=Hfhi2SbCcv7AHUu7atbWNr67wyh6ykTvrScy, ISACTIVE=true, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=2069918539, hash=-1494807473, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], newVal=PROPHETENVIRONMENT [idHash=1282994500, hash=701842127, VERSIONKEY=CM-0050569b1dc6-1592546326608, ACTIVE=null, KEYNAME=rVNTyk8hQI82wMmHtWqUSk7kzGRaWCthbhA0, MODIFICATIONDATE=2020-10-08 14:31:06.23, VALUETYPE=ESKltgbvIcwKDisY5hz8aMcZZVigZblxQvVC, INTERNALVERSION=null, VALUE=Hfhi2SbCcv7AHUu7atbWNr67wyh6ykTvrScy, ISACTIVE=true, NAME=Qqg86pdyUMGAj9Tr9WJroZpeWlLxN2dyUiXZ], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=506206280, hash=1721468689, VALUETYPE=LivePrices, NAME=1SHAM_176], Value:PROPHETENVIRONMENT [idHash=61450569, hash=1905946509, VERSIONKEY=CM-0050569b1dc6-1592504475969, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_176], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=506206280, hash=1721468689, VALUETYPE=LivePrices, NAME=1SHAM_176], newVal=PROPHETENVIRONMENT [idHash=61450569, hash=1905946509, VERSIONKEY=CM-0050569b1dc6-1592504475969, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_176], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=506206280, hash=1721468689, VALUETYPE=LivePrices, NAME=1SHAM_176], Value:PROPHETENVIRONMENT [idHash=61450569, hash=1905946509, VERSIONKEY=CM-0050569b1dc6-1592504475969, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_176], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=506206280, hash=1721468689, VALUETYPE=LivePrices, NAME=1SHAM_176], newVal=PROPHETENVIRONMENT [idHash=61450569, hash=1905946509, VERSIONKEY=CM-0050569b1dc6-1592504475969, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_176], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=719646427, hash=-1552430253, VALUETYPE=LivePrices, NAME=1SHAM_174], Value:PROPHETENVIRONMENT [idHash=2074342611, hash=-1807585031, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_174], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=719646427, hash=-1552430253, VALUETYPE=LivePrices, NAME=1SHAM_174], newVal=PROPHETENVIRONMENT [idHash=2074342611, hash=-1807585031, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_174], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=719646427, hash=-1552430253, VALUETYPE=LivePrices, NAME=1SHAM_174], Value:PROPHETENVIRONMENT [idHash=2074342611, hash=-1807585031, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_174], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,492 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=719646427, hash=-1552430253, VALUETYPE=LivePrices, NAME=1SHAM_174], newVal=PROPHETENVIRONMENT [idHash=2074342611, hash=-1807585031, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.233, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_174], oldVal=null, partCntr=133] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1951506599, hash=189866158, VALUETYPE=LivePrices, NAME=1SHAM_179], Value:PROPHETENVIRONMENT [idHash=788699320, hash=1270078454, VERSIONKEY=CM-0050569b1dc6-1592504475971, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.236, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_179], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1951506599, hash=189866158, VALUETYPE=LivePrices, NAME=1SHAM_179], newVal=PROPHETENVIRONMENT [idHash=788699320, hash=1270078454, VERSIONKEY=CM-0050569b1dc6-1592504475971, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.236, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_179], oldVal=null, partCntr=174] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1951506599, hash=189866158, VALUETYPE=LivePrices, NAME=1SHAM_179], Value:PROPHETENVIRONMENT [idHash=788699320, hash=1270078454, VERSIONKEY=CM-0050569b1dc6-1592504475971, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.236, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_179], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1951506599, hash=189866158, VALUETYPE=LivePrices, NAME=1SHAM_179], newVal=PROPHETENVIRONMENT [idHash=788699320, hash=1270078454, VERSIONKEY=CM-0050569b1dc6-1592504475971, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.236, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_179], oldVal=null, partCntr=174] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=387838560, hash=463794672, VALUETYPE=LivePrices, NAME=1SHAM_694], Value:PROPHETENVIRONMENT [idHash=1842693165, hash=146360123, VERSIONKEY=CM-0050569b1dc6-1592504475698, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.241, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_694], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=387838560, hash=463794672, VALUETYPE=LivePrices, NAME=1SHAM_694], newVal=PROPHETENVIRONMENT [idHash=1842693165, hash=146360123, VERSIONKEY=CM-0050569b1dc6-1592504475698, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.241, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_694], oldVal=null, partCntr=147] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=387838560, hash=463794672, VALUETYPE=LivePrices, NAME=1SHAM_694], Value:PROPHETENVIRONMENT [idHash=1842693165, hash=146360123, VERSIONKEY=CM-0050569b1dc6-1592504475698, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.241, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_694], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=387838560, hash=463794672, VALUETYPE=LivePrices, NAME=1SHAM_694], newVal=PROPHETENVIRONMENT [idHash=1842693165, hash=146360123, VERSIONKEY=CM-0050569b1dc6-1592504475698, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.241, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_694], oldVal=null, partCntr=147] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=86768750, hash=1543175855, VALUETYPE=LivePrices, NAME=1SHAM_189], Value:PROPHETENVIRONMENT [idHash=1269495910, hash=-1467324774, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.245, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_189], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=86768750, hash=1543175855, VALUETYPE=LivePrices, NAME=1SHAM_189], newVal=PROPHETENVIRONMENT [idHash=1269495910, hash=-1467324774, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.245, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_189], oldVal=null, partCntr=161] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=86768750, hash=1543175855, VALUETYPE=LivePrices, NAME=1SHAM_189], Value:PROPHETENVIRONMENT [idHash=1269495910, hash=-1467324774, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.245, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_189], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,493 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=86768750, hash=1543175855, VALUETYPE=LivePrices, NAME=1SHAM_189], newVal=PROPHETENVIRONMENT [idHash=1269495910, hash=-1467324774, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.245, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_189], oldVal=null, partCntr=161] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1017136317, hash=2053710032, VALUETYPE=LivePrices, NAME=1SHAM_188], Value:PROPHETENVIRONMENT [idHash=618719296, hash=-269531396, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.246, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_188], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1017136317, hash=2053710032, VALUETYPE=LivePrices, NAME=1SHAM_188], newVal=PROPHETENVIRONMENT [idHash=618719296, hash=-269531396, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.246, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_188], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1017136317, hash=2053710032, VALUETYPE=LivePrices, NAME=1SHAM_188], Value:PROPHETENVIRONMENT [idHash=618719296, hash=-269531396, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.246, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_188], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1017136317, hash=2053710032, VALUETYPE=LivePrices, NAME=1SHAM_188], newVal=PROPHETENVIRONMENT [idHash=618719296, hash=-269531396, VERSIONKEY=CM-0050569b1dc6-1592504475966, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.246, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_188], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=943581933, hash=-1098641447, VALUETYPE=LivePrices, NAME=1SHAM_190], Value:PROPHETENVIRONMENT [idHash=2068595216, hash=-659562715, VERSIONKEY=CM-0050569b1dc6-1592504475967, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.247, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_190], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=943581933, hash=-1098641447, VALUETYPE=LivePrices, NAME=1SHAM_190], newVal=PROPHETENVIRONMENT [idHash=2068595216, hash=-659562715, VERSIONKEY=CM-0050569b1dc6-1592504475967, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.247, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_190], oldVal=null, partCntr=231] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=943581933, hash=-1098641447, VALUETYPE=LivePrices, NAME=1SHAM_190], Value:PROPHETENVIRONMENT [idHash=2068595216, hash=-659562715, VERSIONKEY=CM-0050569b1dc6-1592504475967, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.247, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_190], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=943581933, hash=-1098641447, VALUETYPE=LivePrices, NAME=1SHAM_190], newVal=PROPHETENVIRONMENT [idHash=2068595216, hash=-659562715, VERSIONKEY=CM-0050569b1dc6-1592504475967, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.247, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_190], oldVal=null, partCntr=231] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1614633850, hash=311413621, VALUETYPE=LivePrices, NAME=1SHAM_183], Value:PROPHETENVIRONMENT [idHash=570769838, hash=802115948, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.25, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_183], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1614633850, hash=311413621, VALUETYPE=LivePrices, NAME=1SHAM_183], newVal=PROPHETENVIRONMENT [idHash=570769838, hash=802115948, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.25, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_183], oldVal=null, partCntr=98] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1614633850, hash=311413621, VALUETYPE=LivePrices, NAME=1SHAM_183], Value:PROPHETENVIRONMENT [idHash=570769838, hash=802115948, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.25, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_183], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,494 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1614633850, hash=311413621, VALUETYPE=LivePrices, NAME=1SHAM_183], newVal=PROPHETENVIRONMENT [idHash=570769838, hash=802115948, VERSIONKEY=CM-0050569b1dc6-1592504475970, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.25, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_183], oldVal=null, partCntr=98] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=389953974, hash=-1344333504, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], Value:PROPHETENVIRONMENT [idHash=330018335, hash=-450680403, VERSIONKEY=CM-0050569b1dc6-1601652240751, ACTIVE=null, KEYNAME=4zYsxWQ5Kd7szgNVXcYG3qB83KN5ns7JT5Dj, MODIFICATIONDATE=2020-10-08 14:31:06.251, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, INTERNALVERSION=null, VALUE=eG1hUbA4SwbyzdVCypAqOHRnwRYTvPgfOtOq, ISACTIVE=true, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=389953974, hash=-1344333504, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], newVal=PROPHETENVIRONMENT [idHash=330018335, hash=-450680403, VERSIONKEY=CM-0050569b1dc6-1601652240751, ACTIVE=null, KEYNAME=4zYsxWQ5Kd7szgNVXcYG3qB83KN5ns7JT5Dj, MODIFICATIONDATE=2020-10-08 14:31:06.251, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, INTERNALVERSION=null, VALUE=eG1hUbA4SwbyzdVCypAqOHRnwRYTvPgfOtOq, ISACTIVE=true, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], oldVal=null, partCntr=203] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=389953974, hash=-1344333504, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], Value:PROPHETENVIRONMENT [idHash=330018335, hash=-450680403, VERSIONKEY=CM-0050569b1dc6-1601652240751, ACTIVE=null, KEYNAME=4zYsxWQ5Kd7szgNVXcYG3qB83KN5ns7JT5Dj, MODIFICATIONDATE=2020-10-08 14:31:06.251, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, INTERNALVERSION=null, VALUE=eG1hUbA4SwbyzdVCypAqOHRnwRYTvPgfOtOq, ISACTIVE=true, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=389953974, hash=-1344333504, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], newVal=PROPHETENVIRONMENT [idHash=330018335, hash=-450680403, VERSIONKEY=CM-0050569b1dc6-1601652240751, ACTIVE=null, KEYNAME=4zYsxWQ5Kd7szgNVXcYG3qB83KN5ns7JT5Dj, MODIFICATIONDATE=2020-10-08 14:31:06.251, VALUETYPE=GhoJ4ov06gxufe9sKJgy1tEOpssDlMrlIqnt, INTERNALVERSION=null, VALUE=eG1hUbA4SwbyzdVCypAqOHRnwRYTvPgfOtOq, ISACTIVE=true, NAME=AI21yt8maoKSwNkRLhUMuYF1NWsKcMdaNxau], oldVal=null, partCntr=203] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=259720474, hash=1664723318, VALUETYPE=LivePrices, NAME=1SHAM_193], Value:PROPHETENVIRONMENT [idHash=978843058, hash=-1575685941, VERSIONKEY=CM-0050569b1dc6-1592504475965, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_193], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=259720474, hash=1664723318, VALUETYPE=LivePrices, NAME=1SHAM_193], newVal=PROPHETENVIRONMENT [idHash=978843058, hash=-1575685941, VERSIONKEY=CM-0050569b1dc6-1592504475965, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_193], oldVal=null, partCntr=181] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=259720474, hash=1664723318, VALUETYPE=LivePrices, NAME=1SHAM_193], Value:PROPHETENVIRONMENT [idHash=978843058, hash=-1575685941, VERSIONKEY=CM-0050569b1dc6-1592504475965, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_193], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=259720474, hash=1664723318, VALUETYPE=LivePrices, NAME=1SHAM_193], newVal=PROPHETENVIRONMENT [idHash=978843058, hash=-1575685941, VERSIONKEY=CM-0050569b1dc6-1592504475965, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_193], oldVal=null, partCntr=181] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1195592090, hash=1892441066, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], Value:PROPHETENVIRONMENT [idHash=266052793, hash=-1703486407, VERSIONKEY=CM-0050569b1dc6-1594631276915, ACTIVE=null, KEYNAME=MP6UgorywOVi3Vcwng03c1FXT5nu4hXiOWgq, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, INTERNALVERSION=null, VALUE=XvwO8xjhKLt2TiI2DepGSdlZhjqVwoR9l80S, ISACTIVE=true, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1195592090, hash=1892441066, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], newVal=PROPHETENVIRONMENT [idHash=266052793, hash=-1703486407, VERSIONKEY=CM-0050569b1dc6-1594631276915, ACTIVE=null, KEYNAME=MP6UgorywOVi3Vcwng03c1FXT5nu4hXiOWgq, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, INTERNALVERSION=null, VALUE=XvwO8xjhKLt2TiI2DepGSdlZhjqVwoR9l80S, ISACTIVE=true, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1195592090, hash=1892441066, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], Value:PROPHETENVIRONMENT [idHash=266052793, hash=-1703486407, VERSIONKEY=CM-0050569b1dc6-1594631276915, ACTIVE=null, KEYNAME=MP6UgorywOVi3Vcwng03c1FXT5nu4hXiOWgq, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, INTERNALVERSION=null, VALUE=XvwO8xjhKLt2TiI2DepGSdlZhjqVwoR9l80S, ISACTIVE=true, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,495 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1195592090, hash=1892441066, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], newVal=PROPHETENVIRONMENT [idHash=266052793, hash=-1703486407, VERSIONKEY=CM-0050569b1dc6-1594631276915, ACTIVE=null, KEYNAME=MP6UgorywOVi3Vcwng03c1FXT5nu4hXiOWgq, MODIFICATIONDATE=2020-10-08 14:31:06.254, VALUETYPE=W1nXRb0Sra6sGVIG6VmX3RB1Tdm1N5j9n3FQ, INTERNALVERSION=null, VALUE=XvwO8xjhKLt2TiI2DepGSdlZhjqVwoR9l80S, ISACTIVE=true, NAME=fwkjO91BYf3j7vvxcB5LKYCFxkCMTjLurvq4], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=338934103, hash=-710586642, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], Value:PROPHETENVIRONMENT [idHash=103603434, hash=-1322891302, VERSIONKEY=CM-0050569b1dc6-1592546326446, ACTIVE=null, KEYNAME=5XeeYKovbhcJZNw5tP3KMJhe4YNxxRqAByUX, MODIFICATIONDATE=2020-10-08 14:31:06.256, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, INTERNALVERSION=null, VALUE=r7U4x3uK9fzHr79hjIQcEpMViV5TyXW2Bk1y, ISACTIVE=true, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=338934103, hash=-710586642, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], newVal=PROPHETENVIRONMENT [idHash=103603434, hash=-1322891302, VERSIONKEY=CM-0050569b1dc6-1592546326446, ACTIVE=null, KEYNAME=5XeeYKovbhcJZNw5tP3KMJhe4YNxxRqAByUX, MODIFICATIONDATE=2020-10-08 14:31:06.256, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, INTERNALVERSION=null, VALUE=r7U4x3uK9fzHr79hjIQcEpMViV5TyXW2Bk1y, ISACTIVE=true, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=338934103, hash=-710586642, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], Value:PROPHETENVIRONMENT [idHash=103603434, hash=-1322891302, VERSIONKEY=CM-0050569b1dc6-1592546326446, ACTIVE=null, KEYNAME=5XeeYKovbhcJZNw5tP3KMJhe4YNxxRqAByUX, MODIFICATIONDATE=2020-10-08 14:31:06.256, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, INTERNALVERSION=null, VALUE=r7U4x3uK9fzHr79hjIQcEpMViV5TyXW2Bk1y, ISACTIVE=true, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=338934103, hash=-710586642, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], newVal=PROPHETENVIRONMENT [idHash=103603434, hash=-1322891302, VERSIONKEY=CM-0050569b1dc6-1592546326446, ACTIVE=null, KEYNAME=5XeeYKovbhcJZNw5tP3KMJhe4YNxxRqAByUX, MODIFICATIONDATE=2020-10-08 14:31:06.256, VALUETYPE=p86w4XKHcgB7lxVe0VU3BioV0Tc7v7m0oZZf, INTERNALVERSION=null, VALUE=r7U4x3uK9fzHr79hjIQcEpMViV5TyXW2Bk1y, ISACTIVE=true, NAME=UHKFM994G5qvd0RyUOIEzb1AMCCAq9hSc6yl], oldVal=null, partCntr=140] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1483662684, hash=-2018194867, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], Value:PROPHETENVIRONMENT [idHash=1693475089, hash=-655531730, VERSIONKEY=CM-0050569b1dc6-1601657790463, ACTIVE=null, KEYNAME=K7BO8AFcFrVP4DQAAD5tYqFBblkRjhZlgyH4, MODIFICATIONDATE=2020-10-08 14:31:06.259, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, INTERNALVERSION=null, VALUE=kE8zBMR61DrqfBtSPUPHjkmiqK88WlJS29kD, ISACTIVE=true, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1483662684, hash=-2018194867, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], newVal=PROPHETENVIRONMENT [idHash=1693475089, hash=-655531730, VERSIONKEY=CM-0050569b1dc6-1601657790463, ACTIVE=null, KEYNAME=K7BO8AFcFrVP4DQAAD5tYqFBblkRjhZlgyH4, MODIFICATIONDATE=2020-10-08 14:31:06.259, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, INTERNALVERSION=null, VALUE=kE8zBMR61DrqfBtSPUPHjkmiqK88WlJS29kD, ISACTIVE=true, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], oldVal=null, partCntr=154] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1483662684, hash=-2018194867, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], Value:PROPHETENVIRONMENT [idHash=1693475089, hash=-655531730, VERSIONKEY=CM-0050569b1dc6-1601657790463, ACTIVE=null, KEYNAME=K7BO8AFcFrVP4DQAAD5tYqFBblkRjhZlgyH4, MODIFICATIONDATE=2020-10-08 14:31:06.259, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, INTERNALVERSION=null, VALUE=kE8zBMR61DrqfBtSPUPHjkmiqK88WlJS29kD, ISACTIVE=true, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,496 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1483662684, hash=-2018194867, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], newVal=PROPHETENVIRONMENT [idHash=1693475089, hash=-655531730, VERSIONKEY=CM-0050569b1dc6-1601657790463, ACTIVE=null, KEYNAME=K7BO8AFcFrVP4DQAAD5tYqFBblkRjhZlgyH4, MODIFICATIONDATE=2020-10-08 14:31:06.259, VALUETYPE=qwG4tJEwJaKswvpf1THTD2V2EMTQHXHqaYIO, INTERNALVERSION=null, VALUE=kE8zBMR61DrqfBtSPUPHjkmiqK88WlJS29kD, ISACTIVE=true, NAME=xI7x6hDUJXwEXg7oXzqmxAE3j1J5h7D1R1X6], oldVal=null, partCntr=154] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,496 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:21,496 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:21,496 [disco-event-worker-#106] WARN org.apache.ignite.internal.managers.discovery.GridDiscoveryManager [] - Node FAILED: TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true] 2020-10-08 14:35:21,496 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:21,496 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:21,496 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:21,496 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1751969694, hash=388737463, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], Value:PROPHETENVIRONMENT [idHash=487294817, hash=1767852729, VERSIONKEY=CM-0050569b1dc6-1592585151222, ACTIVE=null, KEYNAME=fklCXVZMkdjYFqKYUAsid3SHUBtO6K7lun9D, MODIFICATIONDATE=2020-10-08 14:31:06.26, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, INTERNALVERSION=null, VALUE=z6jji6Y4SwOPcFrECOYs7K7L1lwJjUjA9kSl, ISACTIVE=true, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1751969694, hash=388737463, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], newVal=PROPHETENVIRONMENT [idHash=487294817, hash=1767852729, VERSIONKEY=CM-0050569b1dc6-1592585151222, ACTIVE=null, KEYNAME=fklCXVZMkdjYFqKYUAsid3SHUBtO6K7lun9D, MODIFICATIONDATE=2020-10-08 14:31:06.26, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, INTERNALVERSION=null, VALUE=z6jji6Y4SwOPcFrECOYs7K7L1lwJjUjA9kSl, ISACTIVE=true, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], oldVal=null, partCntr=203] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1751969694, hash=388737463, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], Value:PROPHETENVIRONMENT [idHash=487294817, hash=1767852729, VERSIONKEY=CM-0050569b1dc6-1592585151222, ACTIVE=null, KEYNAME=fklCXVZMkdjYFqKYUAsid3SHUBtO6K7lun9D, MODIFICATIONDATE=2020-10-08 14:31:06.26, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, INTERNALVERSION=null, VALUE=z6jji6Y4SwOPcFrECOYs7K7L1lwJjUjA9kSl, ISACTIVE=true, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1751969694, hash=388737463, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], newVal=PROPHETENVIRONMENT [idHash=487294817, hash=1767852729, VERSIONKEY=CM-0050569b1dc6-1592585151222, ACTIVE=null, KEYNAME=fklCXVZMkdjYFqKYUAsid3SHUBtO6K7lun9D, MODIFICATIONDATE=2020-10-08 14:31:06.26, VALUETYPE=kRhsaU7jcqMTwZEzlY9Si2noTBSgeIRWpewT, INTERNALVERSION=null, VALUE=z6jji6Y4SwOPcFrECOYs7K7L1lwJjUjA9kSl, ISACTIVE=true, NAME=2BcmQXFY1GuMhMrh2NLVJrLSWKsXoWtXyagf], oldVal=null, partCntr=203] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=535755551, hash=373866709, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], Value:PROPHETENVIRONMENT [idHash=733050159, hash=1640611927, VERSIONKEY=CM-0050569b1dc6-1601903600415, ACTIVE=null, KEYNAME=P5SIxKftHnSQvGY0YaCtVoZZmIS4WQxCEZjp, MODIFICATIONDATE=2020-10-08 14:31:06.261, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, INTERNALVERSION=null, VALUE=R4L2ZqtzBWqjLe1gK0yqprAWMaW5rIcB7G7M, ISACTIVE=true, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=535755551, hash=373866709, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], newVal=PROPHETENVIRONMENT [idHash=733050159, hash=1640611927, VERSIONKEY=CM-0050569b1dc6-1601903600415, ACTIVE=null, KEYNAME=P5SIxKftHnSQvGY0YaCtVoZZmIS4WQxCEZjp, MODIFICATIONDATE=2020-10-08 14:31:06.261, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, INTERNALVERSION=null, VALUE=R4L2ZqtzBWqjLe1gK0yqprAWMaW5rIcB7G7M, ISACTIVE=true, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=535755551, hash=373866709, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], Value:PROPHETENVIRONMENT [idHash=733050159, hash=1640611927, VERSIONKEY=CM-0050569b1dc6-1601903600415, ACTIVE=null, KEYNAME=P5SIxKftHnSQvGY0YaCtVoZZmIS4WQxCEZjp, MODIFICATIONDATE=2020-10-08 14:31:06.261, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, INTERNALVERSION=null, VALUE=R4L2ZqtzBWqjLe1gK0yqprAWMaW5rIcB7G7M, ISACTIVE=true, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,497 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=535755551, hash=373866709, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], newVal=PROPHETENVIRONMENT [idHash=733050159, hash=1640611927, VERSIONKEY=CM-0050569b1dc6-1601903600415, ACTIVE=null, KEYNAME=P5SIxKftHnSQvGY0YaCtVoZZmIS4WQxCEZjp, MODIFICATIONDATE=2020-10-08 14:31:06.261, VALUETYPE=OizUFIyFv2jQ47NXVE22nWXI0npHnIinKCwS, INTERNALVERSION=null, VALUE=R4L2ZqtzBWqjLe1gK0yqprAWMaW5rIcB7G7M, ISACTIVE=true, NAME=rQBCNmZMAe3itMcuViLf4TsfrgB9wDbxQo1a], oldVal=null, partCntr=196] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=231547263, hash=1659303705, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], Value:PROPHETENVIRONMENT [idHash=291917649, hash=-2098746464, VERSIONKEY=CM-0050569b1dc6-1592546326331, ACTIVE=null, KEYNAME=scqPSFYgV1KxxPAIFKvwToGiBomZ6d5lXnoh, MODIFICATIONDATE=2020-10-08 14:31:06.263, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, INTERNALVERSION=null, VALUE=m1QxyJwz6unuKYwPIna5nLTGJUeRfEJeOmA1, ISACTIVE=true, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=231547263, hash=1659303705, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], newVal=PROPHETENVIRONMENT [idHash=291917649, hash=-2098746464, VERSIONKEY=CM-0050569b1dc6-1592546326331, ACTIVE=null, KEYNAME=scqPSFYgV1KxxPAIFKvwToGiBomZ6d5lXnoh, MODIFICATIONDATE=2020-10-08 14:31:06.263, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, INTERNALVERSION=null, VALUE=m1QxyJwz6unuKYwPIna5nLTGJUeRfEJeOmA1, ISACTIVE=true, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=231547263, hash=1659303705, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], Value:PROPHETENVIRONMENT [idHash=291917649, hash=-2098746464, VERSIONKEY=CM-0050569b1dc6-1592546326331, ACTIVE=null, KEYNAME=scqPSFYgV1KxxPAIFKvwToGiBomZ6d5lXnoh, MODIFICATIONDATE=2020-10-08 14:31:06.263, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, INTERNALVERSION=null, VALUE=m1QxyJwz6unuKYwPIna5nLTGJUeRfEJeOmA1, ISACTIVE=true, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [disco-event-worker-#106] INFO org.apache.ignite.internal.managers.discovery.GridDiscoveryManager [] - Topology snapshot [ver=15, locNode=43e0983b, servers=3, clients=2, state=ACTIVE, CPUs=32, offheap=12.0GB, heap=35.0GB] 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [disco-event-worker-#106] INFO org.apache.ignite.internal.managers.discovery.GridDiscoveryManager [] - ^-- Baseline [id=0, size=3, online=3, offline=0] 2020-10-08 14:35:21,498 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=231547263, hash=1659303705, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], newVal=PROPHETENVIRONMENT [idHash=291917649, hash=-2098746464, VERSIONKEY=CM-0050569b1dc6-1592546326331, ACTIVE=null, KEYNAME=scqPSFYgV1KxxPAIFKvwToGiBomZ6d5lXnoh, MODIFICATIONDATE=2020-10-08 14:31:06.263, VALUETYPE=g0G2MGGpate2aPpyEmCKbPHYrTugF43zUs6T, INTERNALVERSION=null, VALUE=m1QxyJwz6unuKYwPIna5nLTGJUeRfEJeOmA1, ISACTIVE=true, NAME=ERCgHLTD4HsZWUVBS3WmqBbeMdmKNs98hi9v], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1666541946, hash=-982651102, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], Value:PROPHETENVIRONMENT [idHash=245752883, hash=1805052910, VERSIONKEY=CM-0050569b1dc6-1592546326654, ACTIVE=null, KEYNAME=ukQFbYNww1rM1apKFvt1Lw0UtibHWJhmsrb2, MODIFICATIONDATE=2020-10-08 14:31:06.264, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, INTERNALVERSION=null, VALUE=gHHlj8obGoAI93oVVhtsefsR6NnboKlQUYHH, ISACTIVE=true, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1666541946, hash=-982651102, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], newVal=PROPHETENVIRONMENT [idHash=245752883, hash=1805052910, VERSIONKEY=CM-0050569b1dc6-1592546326654, ACTIVE=null, KEYNAME=ukQFbYNww1rM1apKFvt1Lw0UtibHWJhmsrb2, MODIFICATIONDATE=2020-10-08 14:31:06.264, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, INTERNALVERSION=null, VALUE=gHHlj8obGoAI93oVVhtsefsR6NnboKlQUYHH, ISACTIVE=true, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], oldVal=null, partCntr=182] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@212960be java.lang.NullPointerException: null 2020-10-08 14:35:21,498 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1666541946, hash=-982651102, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], Value:PROPHETENVIRONMENT [idHash=245752883, hash=1805052910, VERSIONKEY=CM-0050569b1dc6-1592546326654, ACTIVE=null, KEYNAME=ukQFbYNww1rM1apKFvt1Lw0UtibHWJhmsrb2, MODIFICATIONDATE=2020-10-08 14:31:06.264, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, INTERNALVERSION=null, VALUE=gHHlj8obGoAI93oVVhtsefsR6NnboKlQUYHH, ISACTIVE=true, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,499 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,499 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1666541946, hash=-982651102, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], newVal=PROPHETENVIRONMENT [idHash=245752883, hash=1805052910, VERSIONKEY=CM-0050569b1dc6-1592546326654, ACTIVE=null, KEYNAME=ukQFbYNww1rM1apKFvt1Lw0UtibHWJhmsrb2, MODIFICATIONDATE=2020-10-08 14:31:06.264, VALUETYPE=LhH78yWfpK2Y1saV8BdBQYNzmJNgGWhZKhF0, INTERNALVERSION=null, VALUE=gHHlj8obGoAI93oVVhtsefsR6NnboKlQUYHH, ISACTIVE=true, NAME=9H9QKwW6TcNIQcwvpLOGSUHms0L5PfiXwM2W], oldVal=null, partCntr=182] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,499 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=838885804, hash=1927036088, VALUETYPE=8AvecHObfu3tspsFaSx8i1ZLapV3rot9JFNM, NAME=bXU6DHQ3AmJJIxTNSKsOUW5GVlUXNLUufljA], Value:PROPHETENVIRONMENT [idHash=1119351535, hash=1728428130, VERSIONKEY=CM-0050569b1dc6-1592546326316, ACTIVE=null, KEYNAME=oENfklWHCT4bXOga4Mk8s2dgP3lkUwBOOagf, MODIFICATIONDATE=2020-10-08 14:31:06.266, VALUETYPE=8AvecHObfu3tspsFaSx8i1ZLapV3rot9JFNM, INTERNALVERSION=null, VALUE=f0KJnmW044ntPjjeKx17TzaaJkOhjGmldZOz, ISACTIVE=true, NAME=bXU6DHQ3AmJJIxTNSKsOUW5GVlUXNLUufljA], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,499 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:8AvecHObfu3tspsFaSx8i1ZLapV3rot9JFNM,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,499 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=838885804, hash=1927036088, VALUETYPE=8AvecHObfu3tspsFaSx8i1ZLapV3rot9JFNM, NAME=bXU6DHQ3AmJJIxTNSKsOUW5GVlUXNLUufljA], newVal=PROPHETENVIRONMENT [idHash=1119351535, hash=1728428130, VERSIONKEY=CM-0050569b1dc6-1592546326316, ACTIVE=null, KEYNAME=oENfklWHCT4bXOga4Mk8s2dgP3lkUwBOOagf, MODIFICATIONDATE=2020-10-08 14:31:06.266, VALUETYPE=8AvecHObfu3tspsFaSx8i1ZLapV3rot9JFNM, INTERNALVERSION=null, VALUE=f0KJnmW044ntPjjeKx17TzaaJkOhjGmldZOz, ISACTIVE=true, NAME=bXU6DHQ3AmJJIxTNSKsOUW5GVlUXNLUufljA], oldVal=null, partCntr=154] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=926949628, hash=455334313, VALUETYPE=lTfzk2AF7K3R2MYrICjgqH5AVEFfM5dxxqCm, NAME=k8E4h212Di8YBTyHu1moSeGHbDGPsBev1mdm], Value:PROPHETENVIRONMENT [idHash=585682233, hash=-1911780686, VERSIONKEY=CM-0050569b1dc6-1592546326427, ACTIVE=null, KEYNAME=ItjmBLMmDIc15X7CYt5DKRYJyuQDINhvs79G, MODIFICATIONDATE=2020-10-08 14:31:06.268, VALUETYPE=lTfzk2AF7K3R2MYrICjgqH5AVEFfM5dxxqCm, INTERNALVERSION=null, VALUE=yiD8pBhUf2uJOstTv9u20D6TxViickP9AI3Z, ISACTIVE=true, NAME=k8E4h212Di8YBTyHu1moSeGHbDGPsBev1mdm], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:lTfzk2AF7K3R2MYrICjgqH5AVEFfM5dxxqCm,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=926949628, hash=455334313, VALUETYPE=lTfzk2AF7K3R2MYrICjgqH5AVEFfM5dxxqCm, NAME=k8E4h212Di8YBTyHu1moSeGHbDGPsBev1mdm], newVal=PROPHETENVIRONMENT [idHash=585682233, hash=-1911780686, VERSIONKEY=CM-0050569b1dc6-1592546326427, ACTIVE=null, KEYNAME=ItjmBLMmDIc15X7CYt5DKRYJyuQDINhvs79G, MODIFICATIONDATE=2020-10-08 14:31:06.268, VALUETYPE=lTfzk2AF7K3R2MYrICjgqH5AVEFfM5dxxqCm, INTERNALVERSION=null, VALUE=yiD8pBhUf2uJOstTv9u20D6TxViickP9AI3Z, ISACTIVE=true, NAME=k8E4h212Di8YBTyHu1moSeGHbDGPsBev1mdm], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1593836305, hash=-335209195, VALUETYPE=iL4ZdrHahyvT9DunpDBv3DYepLBBzJPqWso0, NAME=6RorPOhwNDRCV0aVzeBR2quRJc7cICsk9SEX], Value:PROPHETENVIRONMENT [idHash=2142741601, hash=1837968851, VERSIONKEY=CM-0050569b1dc6-1592584814435, ACTIVE=null, KEYNAME=fwZoBcg76QqteiVoPIROJOSWGyfC14pa1OAH, MODIFICATIONDATE=2020-10-08 14:31:06.268, VALUETYPE=iL4ZdrHahyvT9DunpDBv3DYepLBBzJPqWso0, INTERNALVERSION=null, VALUE=R71PfznaRNU2zbfUSLGOAhvOk27k5HTPGlY9, ISACTIVE=true, NAME=6RorPOhwNDRCV0aVzeBR2quRJc7cICsk9SEX], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:iL4ZdrHahyvT9DunpDBv3DYepLBBzJPqWso0,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,500 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=1593836305, hash=-335209195, VALUETYPE=iL4ZdrHahyvT9DunpDBv3DYepLBBzJPqWso0, NAME=6RorPOhwNDRCV0aVzeBR2quRJc7cICsk9SEX], newVal=PROPHETENVIRONMENT [idHash=2142741601, hash=1837968851, VERSIONKEY=CM-0050569b1dc6-1592584814435, ACTIVE=null, KEYNAME=fwZoBcg76QqteiVoPIROJOSWGyfC14pa1OAH, MODIFICATIONDATE=2020-10-08 14:31:06.268, VALUETYPE=iL4ZdrHahyvT9DunpDBv3DYepLBBzJPqWso0, INTERNALVERSION=null, VALUE=R71PfznaRNU2zbfUSLGOAhvOk27k5HTPGlY9, ISACTIVE=true, NAME=6RorPOhwNDRCV0aVzeBR2quRJc7cICsk9SEX], oldVal=null, partCntr=112] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=239028664, hash=-1558883759, VALUETYPE=HNT5NzkIRI3Tu1Jicnb4enITZFA8M07oBDSX, NAME=37XWFgZctkSINwlTVpsVA2B7N1s1NL1T6UOj], Value:PROPHETENVIRONMENT [idHash=1901954382, hash=1012414304, VERSIONKEY=CM-0050569b1dc6-1592546325955, ACTIVE=null, KEYNAME=wPqovoD8xP7yVQd46zJs1gvv8pct0DJQcsOr, MODIFICATIONDATE=2020-10-08 14:31:06.269, VALUETYPE=HNT5NzkIRI3Tu1Jicnb4enITZFA8M07oBDSX, INTERNALVERSION=null, VALUE=b7J6eyTdFlsfxCTg2e1AL7cT3Zr2QdUT9jgW, ISACTIVE=true, NAME=37XWFgZctkSINwlTVpsVA2B7N1s1NL1T6UOj], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:HNT5NzkIRI3Tu1Jicnb4enITZFA8M07oBDSX,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=239028664, hash=-1558883759, VALUETYPE=HNT5NzkIRI3Tu1Jicnb4enITZFA8M07oBDSX, NAME=37XWFgZctkSINwlTVpsVA2B7N1s1NL1T6UOj], newVal=PROPHETENVIRONMENT [idHash=1901954382, hash=1012414304, VERSIONKEY=CM-0050569b1dc6-1592546325955, ACTIVE=null, KEYNAME=wPqovoD8xP7yVQd46zJs1gvv8pct0DJQcsOr, MODIFICATIONDATE=2020-10-08 14:31:06.269, VALUETYPE=HNT5NzkIRI3Tu1Jicnb4enITZFA8M07oBDSX, INTERNALVERSION=null, VALUE=b7J6eyTdFlsfxCTg2e1AL7cT3Zr2QdUT9jgW, ISACTIVE=true, NAME=37XWFgZctkSINwlTVpsVA2B7N1s1NL1T6UOj], oldVal=null, partCntr=175] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=391834928, hash=1037453985, VALUETYPE=7rcO5uV4iyflFq1cdIOwuB8suHuJbn4wn1dc, NAME=kS6Ar0hqHzHLZqsquysHeudjFis9TUS8cU8i], Value:PROPHETENVIRONMENT [idHash=1640098245, hash=-1849698597, VERSIONKEY=CM-0050569b1dc6-1601658030512, ACTIVE=null, KEYNAME=7TNsNcUlILBts6D89GEfxoJtlE52K2SndRlw, MODIFICATIONDATE=2020-10-08 14:31:06.272, VALUETYPE=7rcO5uV4iyflFq1cdIOwuB8suHuJbn4wn1dc, INTERNALVERSION=null, VALUE=yD0FcttbbYzrhLnIy1P3bYbDrbplmDaHOQPh, ISACTIVE=true, NAME=kS6Ar0hqHzHLZqsquysHeudjFis9TUS8cU8i], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:7rcO5uV4iyflFq1cdIOwuB8suHuJbn4wn1dc,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=391834928, hash=1037453985, VALUETYPE=7rcO5uV4iyflFq1cdIOwuB8suHuJbn4wn1dc, NAME=kS6Ar0hqHzHLZqsquysHeudjFis9TUS8cU8i], newVal=PROPHETENVIRONMENT [idHash=1640098245, hash=-1849698597, VERSIONKEY=CM-0050569b1dc6-1601658030512, ACTIVE=null, KEYNAME=7TNsNcUlILBts6D89GEfxoJtlE52K2SndRlw, MODIFICATIONDATE=2020-10-08 14:31:06.272, VALUETYPE=7rcO5uV4iyflFq1cdIOwuB8suHuJbn4wn1dc, INTERNALVERSION=null, VALUE=yD0FcttbbYzrhLnIy1P3bYbDrbplmDaHOQPh, ISACTIVE=true, NAME=kS6Ar0hqHzHLZqsquysHeudjFis9TUS8cU8i], oldVal=null, partCntr=147] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,501 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=4839113, hash=133120787, VALUETYPE=LivePrices, NAME=1SHAM_196], Value:PROPHETENVIRONMENT [idHash=693697598, hash=-1266268805, VERSIONKEY=CM-0050569b1dc6-1592504475955, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.274, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_196], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,502 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:LivePrices,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,502 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=4839113, hash=133120787, VALUETYPE=LivePrices, NAME=1SHAM_196], newVal=PROPHETENVIRONMENT [idHash=693697598, hash=-1266268805, VERSIONKEY=CM-0050569b1dc6-1592504475955, ACTIVE=null, KEYNAME=environment, MODIFICATIONDATE=2020-10-08 14:31:06.274, VALUETYPE=LivePrices, INTERNALVERSION=null, VALUE=DEV, ISACTIVE=true, NAME=1SHAM_196], oldVal=null, partCntr=168] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,502 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=992102017, hash=-1814457449, VALUETYPE=zY9PqhwpvkBCew5n27OwqJ6OY17Fb1m105Bf, NAME=6AMt5hWjeNhDtQ40SYNhn4jYnSfas5ygFmTe], Value:PROPHETENVIRONMENT [idHash=1519949937, hash=-1453122397, VERSIONKEY=CM-0050569b1dc6-1601554310691, ACTIVE=null, KEYNAME=S9MN6tyuhJoEexutzvXBssUjKX5V6Du6Slpq, MODIFICATIONDATE=2020-10-08 14:31:06.278, VALUETYPE=zY9PqhwpvkBCew5n27OwqJ6OY17Fb1m105Bf, INTERNALVERSION=null, VALUE=vawvklnJJDqvAtgcBIXLjlSpFVAKNafN84aU, ISACTIVE=true, NAME=6AMt5hWjeNhDtQ40SYNhn4jYnSfas5ygFmTe], Old ValuenullService:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,502 [sys-stripe-26-#27] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:zY9PqhwpvkBCew5n27OwqJ6OY17Fb1m105Bf,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:21,502 [sys-stripe-26-#27] ERROR com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory [] - PROPHET LISTENS: Remote Filter entryEventCacheContinuousQueryEvent [evtType=CREATED, key=PROPHETENVIRONMENTKEY [idHash=992102017, hash=-1814457449, VALUETYPE=zY9PqhwpvkBCew5n27OwqJ6OY17Fb1m105Bf, NAME=6AMt5hWjeNhDtQ40SYNhn4jYnSfas5ygFmTe], newVal=PROPHETENVIRONMENT [idHash=1519949937, hash=-1453122397, VERSIONKEY=CM-0050569b1dc6-1601554310691, ACTIVE=null, KEYNAME=S9MN6tyuhJoEexutzvXBssUjKX5V6Du6Slpq, MODIFICATIONDATE=2020-10-08 14:31:06.278, VALUETYPE=zY9PqhwpvkBCew5n27OwqJ6OY17Fb1m105Bf, INTERNALVERSION=null, VALUE=vawvklnJJDqvAtgcBIXLjlSpFVAKNafN84aU, ISACTIVE=true, NAME=6AMt5hWjeNhDtQ40SYNhn4jYnSfas5ygFmTe], oldVal=null, partCntr=126] andPredicateMap{VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMapnull,Service:null, Instance:pricingClient, HostName:null,Remotefilterfactory:com.cmcmarkets.prophet.configstore.common.remote.ConfigStoreTableRemoteFilterFactory$1@5e621b9e java.lang.NullPointerException: null 2020-10-08 14:35:21,531 [exchange-worker-#111] INFO org.apache.ignite.internal.exchange.time [] - Started exchange init [topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, customEvt=null, allowMerge=true, exchangeFreeSwitch=false] 2020-10-08 14:35:21,532 [disco-event-worker-#106] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.listener.SegmentationEventListener [] - Received Event 12 Event Node Id 3311a67b-bfcb-41af-8c09-b2e8f2fbde9b for Node : 43e0983b-c8f6-4921-9241-4695ffbfbd0a 2020-10-08 14:35:21,532 [disco-event-worker-#106] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.listener.SegmentationEventListener [] - Event Node Id 3311a67b-bfcb-41af-8c09-b2e8f2fbde9b is not same as local node id: 43e0983b-c8f6-4921-9241-4695ffbfbd0a 2020-10-08 14:35:21,542 [exchange-worker-#111] INFO org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture [] - Finish exchange future [startVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], err=null, rebalanced=true, wasRebalanced=true] 2020-10-08 14:35:21,543 [exchange-worker-#111] INFO org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture [] - Completed partition exchange [localNode=43e0983b-c8f6-4921-9241-4695ffbfbd0a, exchange=GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], evt=NODE_FAILED, evtNode=TcpDiscoveryNode [id=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, consistentId=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b, addrs=ArrayList [0:0:0:0:0:0:0:1, 10.125.215.44, 127.0.0.1], sockAddrs=HashSet [UWSUK2W02064.CMC.LOCAL/10.125.215.44:0, /0:0:0:0:0:0:0:1:0, /127.0.0.1:0], discPort=0, order=12, intOrder=8, lastExchangeTime=1602163619453, loc=false, ver=2.8.1#20200521-sha1:86422096, isClient=true], done=true, newCrdFut=null], topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0]] 2020-10-08 14:35:21,543 [exchange-worker-#111] INFO org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture [] - Exchange timings [startVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], stage="Waiting in exchange queue" (0 ms), stage="Exchange parameters initialization" (0 ms), stage="Determine exchange type" (3 ms), stage="Exchange done" (8 ms), stage="Total time" (11 ms)] 2020-10-08 14:35:21,543 [exchange-worker-#111] INFO org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture [] - Exchange longest local stages [startVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=15, minorTopVer=0]] 2020-10-08 14:35:21,543 [exchange-worker-#111] INFO org.apache.ignite.internal.exchange.time [] - Finished exchange init [topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], crd=false] 2020-10-08 14:35:21,561 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Checkpoint started [checkpointId=9d4308fd-18a5-4bf9-a5a0-52e1a461a40d, startPtr=FileWALPointer [idx=8, fileOff=11308667, len=726221], checkpointBeforeLockTime=108ms, checkpointLockWait=109735ms, checkpointListenersExecuteTime=33ms, checkpointLockHoldTime=38ms, walCpRecordFsyncDuration=5ms, writeCheckpointEntryDuration=4ms, splitAndSortCpPagesDuration=10ms, pages=11092, reason='timeout'] 2020-10-08 14:35:21,563 [exchange-worker-#111] INFO org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager [] - Skipping rebalancing (nothing scheduled) [top=AffinityTopologyVersion [topVer=15, minorTopVer=0], force=false, evt=NODE_FAILED, node=3311a67b-bfcb-41af-8c09-b2e8f2fbde9b] 2020-10-08 14:35:22,270 [sys-stripe-29-#30] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: in update binaryObjectBuilder for table PROPHETENVIRONMENT 2020-10-08 14:35:22,271 [sys-stripe-29-#30] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=417234787, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL] Column VERSIONKEY prophetStoreTableBinaryObject.field(fieldName)->CM-0050569b1dc6-1602163899993,After setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163899993Before setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163657777 2020-10-08 14:35:22,271 [sys-stripe-29-#30] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=417234787, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL] Column VALUE prophetStoreTableBinaryObject.field(fieldName)->abc,After setting binaryObjectBuilder.getField(fieldName)->abcBefore setting binaryObjectBuilder.getField(fieldName)->abc2 2020-10-08 14:35:22,271 [sys-stripe-29-#30] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.Before setting in Cache org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl@6e55eb4 2020-10-08 14:35:22,271 [sys-stripe-29-#30] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=417234787, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Value:PROPHETENVIRONMENT [idHash=236187957, hash=634596907, VERSIONKEY=CM-0050569b1dc6-1602163899993, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.257, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Old ValuePROPHETENVIRONMENT [idHash=697116669, hash=-363829189, VERSIONKEY=CM-0050569b1dc6-1602163657777, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:31:00.871, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc2, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,271 [sys-stripe-29-#30] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,271 [sys-stripe-29-#30] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,271 [sys-stripe-29-#30] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Remote Filter returns false for both value object and old valueobject :( andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=417234787, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Value:PROPHETENVIRONMENT [idHash=236187957, hash=634596907, VERSIONKEY=CM-0050569b1dc6-1602163899993, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.257, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Old ValuePROPHETENVIRONMENT [idHash=697116669, hash=-363829189, VERSIONKEY=CM-0050569b1dc6-1602163657777, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:31:00.871, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc2, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,277 [sys-stripe-22-#23] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: in update binaryObjectBuilder for table PROPHETENVIRONMENT 2020-10-08 14:35:22,277 [sys-stripe-22-#23] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=1562139767, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL] Column VERSIONKEY prophetStoreTableBinaryObject.field(fieldName)->CM-0050569b1dc6-1602163926973,After setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163926973Before setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163899993 2020-10-08 14:35:22,277 [sys-stripe-22-#23] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=1562139767, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL] Column VALUE prophetStoreTableBinaryObject.field(fieldName)->abc2,After setting binaryObjectBuilder.getField(fieldName)->abc2Before setting binaryObjectBuilder.getField(fieldName)->abc 2020-10-08 14:35:22,277 [sys-stripe-22-#23] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.Before setting in Cache org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl@12003219 2020-10-08 14:35:22,277 [sys-stripe-22-#23] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1562139767, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Value:PROPHETENVIRONMENT [idHash=366896283, hash=2057495156, VERSIONKEY=CM-0050569b1dc6-1602163926973, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.272, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc2, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Old ValuePROPHETENVIRONMENT [idHash=1967575466, hash=634596907, VERSIONKEY=CM-0050569b1dc6-1602163899993, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.257, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,277 [sys-stripe-22-#23] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,277 [sys-stripe-22-#23] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,277 [sys-stripe-22-#23] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Remote Filter returns false for both value object and old valueobject :( andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=1562139767, hash=1238780945, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Value:PROPHETENVIRONMENT [idHash=366896283, hash=2057495156, VERSIONKEY=CM-0050569b1dc6-1602163926973, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.272, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc2, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL], Old ValuePROPHETENVIRONMENT [idHash=1967575466, hash=634596907, VERSIONKEY=CM-0050569b1dc6-1602163899993, ACTIVE=null, KEYNAME=ZitzbtSmqhv4lStYv1y3mC16yFFaOdAzRvwf, MODIFICATIONDATE=2020-10-08 14:35:22.257, VALUETYPE=tGZ72UHYJej9vejlypnOzr0Dm6Y1IEAx9EX1, INTERNALVERSION=null, VALUE=abc, ISACTIVE=true, NAME=0A3iyHqVLTKb3tfDrjp3Va0B0B3AnkucxmqL]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,282 [sys-stripe-0-#1] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: in update binaryObjectBuilder for table PROPHETENVIRONMENT 2020-10-08 14:35:22,282 [sys-stripe-0-#1] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=318561553, hash=1462390049, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb] Column VERSIONKEY prophetStoreTableBinaryObject.field(fieldName)->CM-0050569b1dc6-1602163937591,After setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163937591Before setting binaryObjectBuilder.getField(fieldName)->CM-0050569b1dc6-1602163784530 2020-10-08 14:35:22,282 [sys-stripe-0-#1] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.PrimaryKey PROPHETENVIRONMENTKEY [idHash=318561553, hash=1462390049, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb] Column VALUE prophetStoreTableBinaryObject.field(fieldName)->123,After setting binaryObjectBuilder.getField(fieldName)->123Before setting binaryObjectBuilder.getField(fieldName)->1232 2020-10-08 14:35:22,282 [sys-stripe-0-#1] DEBUG com.cmcmarkets.prophet.configstore.helper.ProphetCacheEntryBinaryObjectProcessor [] - Binary CacheEntry Processor: Cache Value Changed.Before setting in Cache org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl@5cde6493 2020-10-08 14:35:22,283 [sys-stripe-0-#1] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Entered Remote Filter . andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=318561553, hash=1462390049, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb], Value:PROPHETENVIRONMENT [idHash=1500836118, hash=1735052954, VERSIONKEY=CM-0050569b1dc6-1602163937591, ACTIVE=null, KEYNAME=gdfQIgx2YpZ2PhmyijEaPvwb3ucgLf70Paec, MODIFICATIONDATE=2020-10-08 14:35:22.279, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, INTERNALVERSION=null, VALUE=123, ISACTIVE=true, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb], Old ValuePROPHETENVIRONMENT [idHash=383465849, hash=-442568172, VERSIONKEY=CM-0050569b1dc6-1602163784530, ACTIVE=null, KEYNAME=gdfQIgx2YpZ2PhmyijEaPvwb3ucgLf70Paec, MODIFICATIONDATE=2020-10-08 14:31:03.069, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, INTERNALVERSION=null, VALUE=1232, ISACTIVE=true, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,283 [sys-stripe-0-#1] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,283 [sys-stripe-0-#1] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: Compare returns false for And Filter predicateMap EntryVALUETYPE=(EQUAL,testvalue), Key:VALUETYPE, Value:NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD,Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:22,283 [sys-stripe-0-#1] INFO com.cmcmarkets.prophet.configstore.common.util.CompareUtil [] - PROPHET LISTENS: PROPHET LISTENS: Remote Filter returns false for both value object and old valueobject :( andPredicateMap {VALUETYPE=(EQUAL,testvalue), KEYNAME=(EQUAL,testkey), NAME=(EQUAL,test1)},orPredicateMap null, key:PROPHETENVIRONMENTKEY [idHash=318561553, hash=1462390049, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb], Value:PROPHETENVIRONMENT [idHash=1500836118, hash=1735052954, VERSIONKEY=CM-0050569b1dc6-1602163937591, ACTIVE=null, KEYNAME=gdfQIgx2YpZ2PhmyijEaPvwb3ucgLf70Paec, MODIFICATIONDATE=2020-10-08 14:35:22.279, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, INTERNALVERSION=null, VALUE=123, ISACTIVE=true, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb], Old ValuePROPHETENVIRONMENT [idHash=383465849, hash=-442568172, VERSIONKEY=CM-0050569b1dc6-1602163784530, ACTIVE=null, KEYNAME=gdfQIgx2YpZ2PhmyijEaPvwb3ucgLf70Paec, MODIFICATIONDATE=2020-10-08 14:31:03.069, VALUETYPE=NTGbpMHeF81JJvwzEPIefPtAbHZALLV91nnD, INTERNALVERSION=null, VALUE=1232, ISACTIVE=true, NAME=0CRkDRXsSIXTk0hPc2nwNcsuHOxQ2NCmohnb]Service:null,Instance:pricingClient, HostName:null 2020-10-08 14:35:23,216 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Checkpoint finished [cpId=9d4308fd-18a5-4bf9-a5a0-52e1a461a40d, pages=11092, markPos=FileWALPointer [idx=8, fileOff=11308667, len=726221], walSegmentsCleared=0, walSegmentsCovered=[7], markDuration=57ms, pagesWrite=131ms, fsync=1524ms, total=111555ms] 2020-10-08 14:35:27,499 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:27,499 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:27,499 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:27,499 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:27,499 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:33,502 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:33,502 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:33,502 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:33,502 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:33,502 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:39,503 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:39,503 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:39,503 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:39,503 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:39,503 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:45,506 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:45,506 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:45,506 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:45,506 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:45,506 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:51,507 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:51,507 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:51,507 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:51,507 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:51,507 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:35:57,509 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:35:57,509 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:35:57,509 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:35:57,509 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:35:57,509 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:03,512 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:03,512 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:03,512 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:03,512 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:03,512 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:09,515 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:09,515 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:09,515 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:09,515 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:09,515 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:15,516 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:15,516 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:15,516 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:15,516 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:15,516 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:21,520 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:21,520 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:21,520 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:21,520 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:21,520 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:27,522 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:27,522 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:27,522 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:27,522 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:27,522 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:33,525 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:33,525 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:33,525 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:33,525 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:33,525 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:39,528 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:39,528 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:39,528 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:39,528 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:39,528 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:45,529 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:45,529 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:45,529 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:45,529 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:45,529 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:51,533 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:51,533 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:51,533 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:51,533 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:51,533 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:36:57,534 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:36:57,534 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:36:57,534 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:36:57,534 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:36:57,534 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:03,536 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:03,536 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:03,536 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:03,536 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:03,536 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:09,538 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:09,538 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:09,538 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:09,538 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:09,538 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:15,540 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:15,540 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:15,540 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:15,540 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:15,540 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:21,542 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:21,542 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:21,542 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:21,542 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:21,542 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:27,544 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:27,544 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:27,544 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:27,544 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:27,544 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:33,545 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:33,545 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:33,545 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:33,545 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:33,545 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:39,547 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:39,547 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:39,547 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:39,547 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:39,547 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:45,551 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:45,551 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:45,551 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:45,551 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:45,551 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:51,553 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:51,553 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:51,553 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:51,553 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:51,553 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:37:57,554 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:37:57,554 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:37:57,554 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:37:57,554 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:37:57,554 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:03,558 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:03,558 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:03,558 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:03,558 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:03,558 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:09,560 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:09,560 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:09,560 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:09,560 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:09,560 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:15,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:15,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:15,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:15,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:15,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:21,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:21,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:21,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:21,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:21,562 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:21,582 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Checkpoint started [checkpointId=09e7bcd8-d074-448a-a220-8a043f9eaea9, startPtr=FileWALPointer [idx=8, fileOff=12184214, len=726221], checkpointBeforeLockTime=39ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=23ms, checkpointLockHoldTime=27ms, walCpRecordFsyncDuration=4ms, writeCheckpointEntryDuration=6ms, splitAndSortCpPagesDuration=0ms, pages=39, reason='timeout'] 2020-10-08 14:38:21,600 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Checkpoint finished [cpId=09e7bcd8-d074-448a-a220-8a043f9eaea9, pages=39, markPos=FileWALPointer [idx=8, fileOff=12184214, len=726221], walSegmentsCleared=0, walSegmentsCovered=[], markDuration=38ms, pagesWrite=0ms, fsync=18ms, total=95ms] 2020-10-08 14:38:27,563 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:27,563 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:27,563 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:27,563 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:27,563 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:33,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:33,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:33,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:33,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:33,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:39,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:39,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:39,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:39,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:39,567 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:45,569 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:45,569 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:45,569 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:45,569 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:45,569 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:51,570 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:51,570 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:51,570 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:51,570 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:51,570 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:38:57,572 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:38:57,572 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:38:57,572 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:38:57,572 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:38:57,572 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:03,574 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:03,574 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:03,574 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:03,574 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:03,574 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:09,577 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:09,577 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:09,577 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:09,577 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:09,577 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:15,579 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:15,579 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:15,579 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:15,579 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:15,579 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:21,581 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:21,581 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:21,581 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:21,581 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:21,581 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:27,583 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:27,583 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:27,583 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:27,583 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:27,583 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:33,584 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:33,584 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:33,584 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:33,584 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:33,584 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:39,585 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:39,585 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:39,585 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:39,585 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:39,585 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:45,589 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:45,589 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:45,589 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:45,589 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:45,589 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:51,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:51,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:51,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:51,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:51,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:39:57,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:39:57,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:39:57,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:39:57,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:39:57,592 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:03,597 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:03,597 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:03,597 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:03,597 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:03,597 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:09,598 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:09,598 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:09,598 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:09,598 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:09,598 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:15,599 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:15,599 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:15,599 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:15,599 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:15,599 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:21,601 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:21,601 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:21,601 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:21,601 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:21,601 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:27,602 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:27,602 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:27,602 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:27,602 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:27,602 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:33,604 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:33,604 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:33,604 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:33,604 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:33,604 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:39,607 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:39,607 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:39,607 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:39,607 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:39,607 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:45,609 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:45,609 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:45,609 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:45,609 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:45,609 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:51,611 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:51,611 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:51,611 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:51,611 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:51,611 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:40:57,613 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:40:57,613 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:40:57,613 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:40:57,613 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:40:57,613 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:03,614 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:03,614 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:03,614 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:03,614 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:03,614 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:09,618 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:09,618 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:09,618 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:09,618 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:09,618 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:15,621 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:15,621 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:15,621 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:15,621 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:15,621 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:21,608 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=35ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=27ms, checkpointLockHoldTime=29ms, reason='timeout'] 2020-10-08 14:41:21,623 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:21,623 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:21,623 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:21,623 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:21,623 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:27,625 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:27,625 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:27,625 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:27,625 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:27,625 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:33,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:33,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:33,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:33,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:33,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:39,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:39,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:39,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:39,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:39,627 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:45,629 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:45,629 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:45,629 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:45,629 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:45,629 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:51,632 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:51,632 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:51,632 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:51,632 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:51,632 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:41:57,633 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:41:57,633 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:41:57,633 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:41:57,633 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:41:57,633 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:03,635 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:03,635 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:03,635 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:03,635 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:03,635 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:09,636 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:09,636 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:09,636 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:09,636 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:09,636 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:15,639 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:15,639 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:15,639 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:15,639 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:15,639 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:21,640 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:21,640 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:21,640 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:21,640 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:21,640 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:27,642 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:27,642 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:27,642 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:27,642 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:27,642 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:33,644 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:33,644 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:33,644 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:33,644 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:33,644 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:39,646 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:39,646 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:39,646 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:39,646 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:39,646 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:45,649 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:45,649 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:45,649 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:45,649 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:45,649 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:51,651 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:51,651 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:51,651 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:51,651 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:51,651 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:42:57,652 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:42:57,652 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:42:57,652 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:42:57,652 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:42:57,652 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:03,654 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:03,654 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:03,654 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:03,654 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:03,654 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:09,655 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:09,655 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:09,655 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:09,655 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:09,655 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:15,657 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:15,657 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:15,657 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:15,657 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:15,657 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:21,658 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:21,658 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:21,658 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:21,658 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:21,658 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:27,660 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:27,660 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:27,660 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:27,660 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:27,660 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:33,664 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:33,664 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:33,664 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:33,664 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:33,664 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:39,665 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:39,665 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:39,665 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:39,665 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:39,665 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:45,668 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:45,668 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:45,668 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:45,668 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:45,668 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:51,669 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:51,669 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:51,669 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:51,669 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:51,669 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:43:57,671 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:43:57,671 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:43:57,671 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:43:57,671 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:43:57,671 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:03,672 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:03,672 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:03,672 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:03,672 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:03,672 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:09,674 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:09,674 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:09,674 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:09,674 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:09,674 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:15,676 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:15,676 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:15,676 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:15,676 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:15,676 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:21,643 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=33ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=27ms, checkpointLockHoldTime=31ms, reason='timeout'] 2020-10-08 14:44:21,677 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:21,677 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:21,677 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:21,677 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:21,677 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:27,679 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:27,679 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:27,679 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:27,679 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:27,679 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:33,681 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:33,681 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:33,681 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:33,681 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:33,681 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:39,683 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:39,683 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:39,683 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:39,683 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:39,683 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:45,685 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:45,685 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:45,685 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:45,685 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:45,685 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:51,687 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:51,687 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:51,687 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:51,687 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:51,687 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:44:57,689 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:44:57,689 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:44:57,689 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:44:57,689 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:44:57,689 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:03,690 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:03,690 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:03,690 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:03,690 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:03,690 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:09,691 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:09,691 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:09,691 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:09,691 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:09,691 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:15,693 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:15,693 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:15,693 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:15,693 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:15,693 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:21,694 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:21,694 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:21,694 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:21,694 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:21,694 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:27,696 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:27,696 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:27,696 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:27,696 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:27,696 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:33,698 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:33,698 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:33,698 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:33,698 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:33,698 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:39,699 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:39,699 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:39,699 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:39,699 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:39,699 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:45,703 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:45,703 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:45,703 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:45,703 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:45,703 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:51,705 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:51,705 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:51,705 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:51,705 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:51,705 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:45:57,706 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:45:57,706 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:45:57,706 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:45:57,706 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:45:57,706 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:03,707 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:03,707 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:03,707 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:03,707 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:03,707 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:09,710 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:09,710 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:09,710 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:09,710 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:09,710 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:15,711 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:15,711 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:15,711 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:15,711 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:15,711 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:21,713 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:21,713 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:21,713 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:21,713 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:21,713 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:27,714 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:27,714 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:27,714 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:27,714 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:27,714 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:33,716 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:33,716 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:33,716 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:33,716 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:33,716 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:39,717 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:39,717 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:39,717 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:39,717 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:39,717 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:45,719 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:45,719 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:45,719 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:45,719 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:45,719 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:51,720 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:51,720 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:51,720 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:51,720 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:51,720 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:46:57,721 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:46:57,721 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:46:57,721 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:46:57,721 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:46:57,721 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:03,722 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:03,722 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:03,722 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:03,722 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:03,722 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:09,723 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:09,723 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:09,723 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:09,723 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:09,723 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:15,725 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:15,725 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:15,725 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:15,725 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:15,725 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:21,677 [db-checkpoint-thread-#147] INFO org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager [] - Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=35ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=26ms, checkpointLockHoldTime=28ms, reason='timeout'] 2020-10-08 14:47:21,728 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:21,728 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:21,728 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:21,728 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:21,728 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:27,730 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:27,730 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:27,730 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:27,730 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:27,730 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:33,734 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:33,734 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:33,734 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:33,734 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:33,734 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:39,736 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:39,736 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:39,736 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:39,736 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:39,736 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:45,737 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:45,737 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:45,737 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:45,737 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:45,737 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:51,739 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:51,739 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:51,739 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:51,739 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:51,739 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:47:57,740 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:47:57,740 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:47:57,740 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:47:57,740 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:47:57,740 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:03,742 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:03,742 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:03,742 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:03,742 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:03,742 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:09,744 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:09,744 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:09,744 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:09,744 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:09,744 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:15,746 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:15,746 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:15,746 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:15,746 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:15,746 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:21,748 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:21,748 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:21,748 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:21,748 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:21,748 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:27,750 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:27,750 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:27,750 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:27,750 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:27,750 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:33,752 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:33,752 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:33,752 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:33,752 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:33,752 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:39,756 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:39,756 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:39,756 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationResolver isValidSegment quorum check passed. Quorum 2 Current Size 3, configuredIpSize 3 2020-10-08 14:48:39,756 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segment resolver processed [resolver=QuorumCheckSegmentationResolver, attempt=0, valid segment=true] 2020-10-08 14:48:39,756 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Segmentation checks finished successfully. 2020-10-08 14:48:45,758 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment checks now. 2020-10-08 14:48:45,758 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.SegmentationProcessor [] - Starting segment resolver [resolver= QuorumCheckSegmentationResolver, attempt=0] 2020-10-08 14:48:45,758 [disco-net-seg-chk-worker-#105] DEBUG com.cmcmarkets.prophet.configstore.common.plugin.segmentation.resolver.QuorumCheckSegmentationResolver [] - QuorumCheckSegmentationR