[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 18 - Unstable

classic Classic list List threaded Threaded
7 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 18 - Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/18/

4 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([4B35E62A10EB399D:C01235FB51ED9219]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
        at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=2808, name=qtp389115733-2808-acceptor-0@24998c06-ServerConnector@55f2879{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35723}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2865, name=Scheduler-358153316, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2809, name=qtp389115733-2809, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2810, name=qtp389115733-2810, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2816, name=Scheduler-2057414029, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=2807, name=qtp389115733-2807, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=2806, name=qtp389115733-2806, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=2815, name=qtp389115733-2815, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=2814, name=qtp389115733-2814, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=2811, name=qtp389115733-2811, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
   1) Thread[id=2808, name=qtp389115733-2808-acceptor-0@24998c06-ServerConnector@55f2879{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35723}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2865, name=Scheduler-358153316, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2809, name=qtp389115733-2809, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2810, name=qtp389115733-2810, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2816, name=Scheduler-2057414029, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=2807, name=qtp389115733-2807, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=2806, name=qtp389115733-2806, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=2815, name=qtp389115733-2815, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=2814, name=qtp389115733-2814, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=2811, name=qtp389115733-2811, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([4B35E62A10EB399D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2865, name=Scheduler-358153316, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2816, name=Scheduler-2057414029, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2807, name=qtp389115733-2807, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2806, name=qtp389115733-2806, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2865, name=Scheduler-358153316, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2816, name=Scheduler-2057414029, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2807, name=qtp389115733-2807, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2806, name=qtp389115733-2806, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$52/2144957121.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([4B35E62A10EB399D]:0)


FAILED:  org.apache.solr.common.cloud.TestCollectionStateWatchers.testWatchesWorkForStateFormat1

Error Message:
CollectionStateWatcher not notified of stateformat=1 collection creation

Stack Trace:
java.lang.AssertionError: CollectionStateWatcher not notified of stateformat=1 collection creation
        at __randomizedtesting.SeedInfo.seed([1105111071245BDC:76C20DFE859C7F7D]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.assertTrue(Assert.java:41)
        at org.apache.solr.common.cloud.TestCollectionStateWatchers.testWatchesWorkForStateFormat1(TestCollectionStateWatchers.java:304)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13872 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.ShardSplitTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/init-core-data-001
   [junit4]   2> 480352 INFO  (SUITE-ShardSplitTest-seed#[4B35E62A10EB399D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 480386 INFO  (SUITE-ShardSplitTest-seed#[4B35E62A10EB399D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 480387 INFO  (SUITE-ShardSplitTest-seed#[4B35E62A10EB399D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 480387 INFO  (SUITE-ShardSplitTest-seed#[4B35E62A10EB399D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 480406 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 480441 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 480441 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 480547 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer start zk server on port:39522
   [junit4]   2> 480547 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39522
   [junit4]   2> 480547 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39522
   [junit4]   2> 480632 INFO  (zkConnectionManagerCallback-942-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 480693 INFO  (zkConnectionManagerCallback-944-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 480721 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 480722 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 480724 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 480725 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 480742 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 480743 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 480744 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 480745 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 480762 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 480763 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 480764 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 480765 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 481647 WARN  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 481647 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 481647 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 481647 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 481706 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 481706 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 481706 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 481708 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d8b828c{/,null,AVAILABLE}
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@75345eb4{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38163}
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.s.Server Started @481851ms
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=38163, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/control-001/cores}
   [junit4]   2> 481731 ERROR (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 481731 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-13T08:18:11.005Z
   [junit4]   2> 481768 INFO  (zkConnectionManagerCallback-946-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 481769 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 481769 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/control-001/solr.xml
   [junit4]   2> 481802 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 481802 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 481836 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4af1aa6f, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 483788 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 483802 WARN  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@c3caee0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 483821 WARN  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@32a22a24[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 483830 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39522/solr
   [junit4]   2> 483849 INFO  (zkConnectionManagerCallback-952-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 483895 INFO  (zkConnectionManagerCallback-954-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 484264 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38163_
   [junit4]   2> 484264 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer Overseer (id=74338410066673668-127.0.0.1:38163_-n_0000000000) starting
   [junit4]   2> 484352 INFO  (zkConnectionManagerCallback-961-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 484434 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39522/solr ready
   [junit4]   2> 484458 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 484459 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38163_
   [junit4]   2> 484460 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0 #Completed tasks: 0
   [junit4]   2> 484461 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 484461 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 484461 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 484461 INFO  (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 484477 INFO  (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38163_
   [junit4]   2> 484514 INFO  (zkCallback-960-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 484542 INFO  (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 484644 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 484792 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4af1aa6f
   [junit4]   2> 484829 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4af1aa6f
   [junit4]   2> 484829 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4af1aa6f
   [junit4]   2> 484846 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [n:127.0.0.1:38163_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/control-001/cores
   [junit4]   2> 485227 INFO  (zkConnectionManagerCallback-967-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 485328 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 485422 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39522/solr ready
   [junit4]   2> 485447 INFO  (qtp1712227861-2661) [n:127.0.0.1:38163_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38163_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 485496 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
   [junit4]   2> 485496 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
   [junit4]   2> 485496 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:38163_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 485511 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
   [junit4]   2> 485511 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:38163_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 485511 INFO  (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 485531 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 0
   [junit4]   2> 485531 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 485531 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 485531 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 485531 INFO  (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 485532 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/control_collection
   [junit4]   2> 485532 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:control_collection
   [junit4]   2> 485533 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 485534 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:38163_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 485534 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 485584 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/control_collection/state.json
   [junit4]   2> 485658 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection control_collection, shardNames [shard1] , message : {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:38163_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 485659 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Creating core control_collection_shard1_replica_n1 as part of shard shard1 of collection control_collection on 127.0.0.1:38163_
   [junit4]   2> 485678 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 485678 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 485766 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:38163",
   [junit4]   2>   "node_name":"127.0.0.1:38163_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 485766 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:38163",
   [junit4]   2>   "node_name":"127.0.0.1:38163_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 485767 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:38163","node_name":"127.0.0.1:38163_","state":"down","type":"NRT"}
   [junit4]   2> 485768 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/0)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:38163",
   [junit4]   2>           "node_name":"127.0.0.1:38163_",
   [junit4]   2>           "state":"down",
   [junit4]   2>           "type":"NRT"}}}}}
   [junit4]   2> 485888 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 0
   [junit4]   2> 486904 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 486976 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 487471 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 487545 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 487597 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 0
   [junit4]   2> 487597 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 487597 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 487598 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 487598 INFO  (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 487638 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 487639 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4af1aa6f
   [junit4]   2> 487712 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_4B35E62A10EB399D-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 487728 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1801511683, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 487755 WARN  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 487987 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 487987 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 487990 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 487990 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 488015 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=66.98046875, floorSegmentMB=1.4150390625, forceMergeDeletesPctAllowed=23.19556065168484, segmentsPerTier=37.0, maxCFSSegmentSizeMB=0.6689453125, noCFSRatio=0.4450691063830723, deletesPctAllowed=23.241872948317926
   [junit4]   2> 488016 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30f85334[control_collection_shard1_replica_n1] main]
   [junit4]   2> 488036 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 488036 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 488037 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 488037 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1627877641877979136
   [junit4]   2> 488093 INFO  (searcherExecutor-671-thread-1-processing-n:127.0.0.1:38163_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@30f85334[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 488109 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 488109 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 488112 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 488112 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 488112 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38163/control_collection_shard1_replica_n1/
   [junit4]   2> 488124 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 488129 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38163/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 488129 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/74338410066673668-core_node2-n_0000000000
   [junit4]   2> 488131 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38163/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 488154 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:38163",
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "state":"active"} current state version: 0
   [junit4]   2> 488154 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 1
   [junit4]   2> 488155 INFO  (zkCallback-953-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 488155 INFO  (zkCallback-953-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 488161 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 488169 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:38163",
   [junit4]   2>   "node_name":"127.0.0.1:38163_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 488169 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:38163",
   [junit4]   2>   "node_name":"127.0.0.1:38163_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 488169 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:38163","node_name":"127.0.0.1:38163_","state":"active","type":"NRT","leader":"true"}
   [junit4]   2> 488169 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/2)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:38163",
   [junit4]   2>           "node_name":"127.0.0.1:38163_",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true"}}}}}
   [junit4]   2> 488252 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2574
   [junit4]   2> 488252 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Finished create command on all shards for collection: control_collection
   [junit4]   2> 488252 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 488283 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 2
   [junit4]   2> 488284 INFO  (zkCallback-953-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 488284 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
   [junit4]   2> 488284 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 488284 INFO  (zkCallback-953-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 488286 INFO  (qtp1712227861-2661) [n:127.0.0.1:38163_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 488286 INFO  (qtp1712227861-2661) [n:127.0.0.1:38163_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38163_&wt=javabin&version=2} status=0 QTime=2839
   [junit4]   2> 488339 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 488339 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 488340 INFO  (zkCallback-953-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 488340 INFO  (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 488340 DEBUG (OverseerThreadFactory-666-thread-1-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{success={127.0.0.1:38163_={responseHeader={status=0,QTime=2574},core=control_collection_shard1_replica_n1}}}
   [junit4]   2> 488525 INFO  (zkConnectionManagerCallback-973-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 488539 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 488540 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39522/solr ready
   [junit4]   2> 488540 INFO  (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[4B35E62A10EB399D]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 488560 INFO  (qtp1712227861-2663) [n:127.0.0.1:38163_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 488605 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
   [junit4]   2> 488605 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
   [junit4]   2> 488605 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 488621 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 1
   [junit4]   2> 488621 INFO  (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 488621 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 488621 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 488621 DEBUG (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 488621 INFO  (OverseerCollectionConfigSetProcessor-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 488631 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
   [junit4]   2> 488631 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 488631 INFO  (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 488632 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1
   [junit4]   2> 488632 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:collection1
   [junit4]   2> 488632 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 488634 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 488634 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 488735 DEBUG (OverseerStateUpdate-74338410066673668-127.0.0.1:38163_-n_0000000000) [n:127.0.0.1:38163_    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 488789 WARN  (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 488790 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.a.c.CreateCollectionCmd Finished create command for collection: collection1
   [junit4]   2> 488790 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 488805 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as completed.
   [junit4]   2> 488805 DEBUG (OverseerThreadFactory-666-thread-2-processing-n:127.0.0.1:38163_) [n:127.0.0.1:38163_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 488805 DEBUG (OverseerThreadFactory-666-thread-2-

[...truncated too long message...]

source] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null245461461
     [copy] Copying 236 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null245461461
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null245461461

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 34ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 479 minutes 31 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 19 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/19/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MMapDirectory, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2106)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2265)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1108)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:995)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)   expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MMapDirectory, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)  at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2106)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2265)  at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1108)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:995)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MMapDirectory, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)
        at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2106)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2265)
        at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1108)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:995)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MMapDirectory, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)
        at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2106)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2265)
        at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1108)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:995)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:695)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

>
        at __randomizedtesting.SeedInfo.seed([83603EC0FA962B10]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.failNotNull(Assert.java:755)
        at org.junit.Assert.assertNull(Assert.java:737)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:322)
        at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14406 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/init-core-data-001
   [junit4]   2> 1842012 INFO  (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[83603EC0FA962B10]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ff_zaj/uu
   [junit4]   2> 1842012 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ChaosMonkeySafeLeaderWithPullReplicasTest Starting ChaosMonkey test with 1 shards and 4 nodes
   [junit4]   2> 1842015 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1842015 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1842016 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1842115 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer start zk server on port:42000
   [junit4]   2> 1842116 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:42000
   [junit4]   2> 1842116 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 42000
   [junit4]   2> 1842148 INFO  (zkConnectionManagerCallback-7439-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842168 INFO  (zkConnectionManagerCallback-7441-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842169 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1842170 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1842207 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1842208 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1842209 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1842210 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1842211 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1842213 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1842214 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1842215 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1842216 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1842217 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1842616 WARN  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1842616 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1842616 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1842616 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1842672 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1842672 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1842672 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1842675 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52f72c45{/ff_zaj/uu,null,AVAILABLE}
   [junit4]   2> 1842675 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@25d1ae32{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:33825}
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.s.Server Started @1842731ms
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/tempDir-001/control/data, hostContext=/ff_zaj/uu, hostPort=33825, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/control-001/cores}
   [junit4]   2> 1842676 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1842676 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-14T06:30:05.446Z
   [junit4]   2> 1842688 INFO  (zkConnectionManagerCallback-7443-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842689 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1842689 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/control-001/solr.xml
   [junit4]   2> 1842694 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1842694 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1842696 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1842751 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1842753 WARN  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4c24cbbd[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1842757 WARN  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@41f387ea[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1842758 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42000/solr
   [junit4]   2> 1842776 INFO  (zkConnectionManagerCallback-7449-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842800 INFO  (zkConnectionManagerCallback-7451-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842893 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33825_ff_zaj%2Fuu
   [junit4]   2> 1842894 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.Overseer Overseer (id=72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) starting
   [junit4]   2> 1842932 INFO  (zkConnectionManagerCallback-7458-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1842937 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1842940 INFO  (OverseerStateUpdate-72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33825_ff_zaj%2Fuu
   [junit4]   2> 1842947 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33825_ff_zaj%2Fuu
   [junit4]   2> 1842949 INFO  (OverseerStateUpdate-72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1842981 DEBUG (OverseerAutoScalingTriggerThread-72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: []
   [junit4]   2> 1842983 DEBUG (ScheduledTrigger-8020-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 0 and last live nodes: 0
   [junit4]   2> 1843008 INFO  (zkCallback-7457-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1843072 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1843134 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1843199 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1843200 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1843202 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/control-001/cores
   [junit4]   2> 1843376 INFO  (zkConnectionManagerCallback-7464-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1843377 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1843378 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1843392 INFO  (qtp1391128725-30804) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33825_ff_zaj%252Fuu&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1843403 INFO  (OverseerThreadFactory-8022-thread-1-processing-n:127.0.0.1:33825_ff_zaj%2Fuu) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1843537 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1843538 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1843985 DEBUG (ScheduledTrigger-8020-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 0
   [junit4]   2> 1844557 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1844632 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1844827 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1844867 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1844867 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1844868 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1844871 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=45.234375, floorSegmentMB=1.919921875, forceMergeDeletesPctAllowed=13.6802821326066, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7850267212849833, deletesPctAllowed=25.312262654985837
   [junit4]   2> 1844929 WARN  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1844986 DEBUG (ScheduledTrigger-8020-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1845160 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1845160 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1845163 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
   [junit4]   2> 1845163 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1845165 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=13, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1845165 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2052db41[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1845167 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1845168 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1845168 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1845168 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1627961434243596288
   [junit4]   2> 1845172 INFO  (searcherExecutor-8027-thread-1-processing-n:127.0.0.1:33825_ff_zaj%2Fuu x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2052db41[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1845176 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1845176 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33825/ff_zaj/uu/control_collection_shard1_replica_n1/
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:33825/ff_zaj/uu/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1845195 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72954110937726980-core_node2-n_0000000000
   [junit4]   2> 1845198 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33825/ff_zaj/uu/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1845300 INFO  (zkCallback-7450-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1845302 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1845303 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1766
   [junit4]   2> 1845304 INFO  (zkCallback-7450-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1845307 INFO  (qtp1391128725-30804) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1845415 INFO  (zkCallback-7450-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1845415 INFO  (zkCallback-7450-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1845415 INFO  (OverseerCollectionConfigSetProcessor-72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1845416 INFO  (qtp1391128725-30804) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33825_ff_zaj%252Fuu&wt=javabin&version=2} status=0 QTime=2023
   [junit4]   2> 1845440 INFO  (zkCallback-7450-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1845492 INFO  (zkConnectionManagerCallback-7470-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1845494 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1845495 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1845495 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1845496 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1845512 INFO  (OverseerThreadFactory-8022-thread-2-processing-n:127.0.0.1:33825_ff_zaj%2Fuu) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1845718 WARN  (OverseerThreadFactory-8022-thread-2-processing-n:127.0.0.1:33825_ff_zaj%2Fuu) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1845720 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1845721 INFO  (qtp1391128725-30806) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=224
   [junit4]   2> 1845731 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=3 numOtherReplicas=1
   [junit4]   2> 1845986 DEBUG (ScheduledTrigger-8020-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1846340 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-1-001 of type NRT
   [junit4]   2> 1846368 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1846368 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1846368 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1846369 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1846496 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1846496 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1846496 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1846508 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@37106f4{/ff_zaj/uu,null,AVAILABLE}
   [junit4]   2> 1846508 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4c2fac62{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:36975}
   [junit4]   2> 1846508 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.Server Started @1846564ms
   [junit4]   2> 1846508 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/ff_zaj/uu, hostPort=36975, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-1-001/cores}
   [junit4]   2> 1846509 ERROR (closeThreadPool-7471-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1846509 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1846509 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1846509 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1846509 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1846509 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-14T06:30:09.279Z
   [junit4]   2> 1846540 INFO  (zkConnectionManagerCallback-7473-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1846545 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1846545 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-1-001/solr.xml
   [junit4]   2> 1846550 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1846550 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1846613 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1846986 DEBUG (ScheduledTrigger-8020-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1846995 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-2-001 of type PULL
   [junit4]   2> 1847000 WARN  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1847000 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1847000 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1847001 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1847060 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1847060 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1847060 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1847070 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63a02c53{/ff_zaj/uu,null,AVAILABLE}
   [junit4]   2> 1847070 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@62278082{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41762}
   [junit4]   2> 1847070 INFO  (closeThreadPool-7471-thread-2) [    ] o.e.j.s.Server Started @1847126ms
   [junit4]   2> 1847070 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/tempDir-001/jetty2, replicaType=PULL, solrconfig=solrconfig.xml, hostContext=/ff_zaj/uu, hostPort=41762, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-2-001/cores}
   [junit4]   2> 1847071 ERROR (closeThreadPool-7471-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1847071 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1847071 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1847071 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1847071 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1847071 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-14T06:30:09.841Z
   [junit4]   2> 1847084 INFO  (zkConnectionManagerCallback-7475-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847085 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1847085 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-2-001/solr.xml
   [junit4]   2> 1847089 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1847089 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1847091 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1847349 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1847357 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4206d095[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1847381 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@493908e6[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1847382 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42000/solr
   [junit4]   2> 1847418 INFO  (zkConnectionManagerCallback-7481-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847444 INFO  (zkConnectionManagerCallback-7483-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847449 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1847454 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.ZkController Publish node=127.0.0.1:36975_ff_zaj%2Fuu as DOWN
   [junit4]   2> 1847454 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1847454 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36975_ff_zaj%2Fuu
   [junit4]   2> 1847456 INFO  (zkCallback-7450-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1847481 INFO  (zkCallback-7469-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1847496 INFO  (zkCallback-7457-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1847498 INFO  (zkCallback-7482-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1847506 INFO  (OverseerCollectionConfigSetProcessor-72954110937726980-127.0.0.1:33825_ff_zaj%2Fuu-n_0000000000) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1847535 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-3-001 of type PULL
   [junit4]   2> 1847552 WARN  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1847552 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1847552 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1847553 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1847557 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1847557 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1847557 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1847557 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f3d8371{/ff_zaj/uu,null,AVAILABLE}
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6d2870b6{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34343}
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.e.j.s.Server Started @1847614ms
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/tempDir-001/jetty3, replicaType=PULL, solrconfig=solrconfig.xml, hostContext=/ff_zaj/uu, hostPort=34343, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-3-001/cores}
   [junit4]   2> 1847559 ERROR (closeThreadPool-7471-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1847559 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-14T06:30:10.329Z
   [junit4]   2> 1847561 INFO  (zkConnectionManagerCallback-7490-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847562 INFO  (zkConnectionManagerCallback-7492-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847563 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1847563 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1847563 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-3-001/solr.xml
   [junit4]   2> 1847564 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1847576 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1847579 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1847579 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1847581 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1847601 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1847602 WARN  (closeThreadPool-7471-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2664d8e5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1847677 WARN  (closeThreadPool-7471-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@474e0fb3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1847678 INFO  (closeThreadPool-7471-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42000/solr
   [junit4]   2> 1847720 INFO  (zkConnectionManagerCallback-7499-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847730 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1847764 INFO  (zkConnectionManagerCallback-7501-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847772 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1847776 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.ZkController Publish node=127.0.0.1:41762_ff_zaj%2Fuu as DOWN
   [junit4]   2> 1847777 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1847777 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41762_ff_zaj%2Fuu
   [junit4]   2> 1847778 INFO  (zkCallback-7457-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847778 INFO  (zkCallback-7450-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847778 INFO  (zkCallback-7469-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847792 INFO  (zkCallback-7489-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847792 INFO  (zkCallback-7482-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847808 INFO  (zkCallback-7500-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1847864 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1847865 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1847866 INFO  (closeThreadPool-7471-thread-1) [n:127.0.0.1:36975_ff_zaj%2Fuu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-1-001/cores
   [junit4]   2> 1847952 INFO  (zkConnectionManagerCallback-7508-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1847953 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1847954 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1847980 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1847986 DEBUG (ScheduledTrigger-8020-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 1
   [junit4]   2> 1848109 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1848111 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848149 WARN  (closeThreadPool-7471-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@258a9b35[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1848225 WARN  (closeThreadPool-7471-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4d27ccbd[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1848226 INFO  (closeThreadPool-7471-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42000/solr
   [junit4]   2> 1848232 INFO  (zkConnectionManagerCallback-7515-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848232 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848232 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848304 INFO  (closeThreadPool-7471-thread-2) [n:127.0.0.1:41762_ff_zaj%2Fuu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-2-001/cores
   [junit4]   2> 1848324 INFO  (zkConnectionManagerCallback-7517-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848330 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1848354 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.ZkController Publish node=127.0.0.1:34343_ff_zaj%2Fuu as DOWN
   [junit4]   2> 1848355 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1848355 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34343_ff_zaj%2Fuu
   [junit4]   2> 1848357 INFO  (zkCallback-7469-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848357 INFO  (zkCallback-7489-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848357 INFO  (zkCallback-7450-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848360 INFO  (zkCallback-7482-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848360 INFO  (zkCallback-7457-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848366 INFO  (zkCallback-7500-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848376 INFO  (zkCallback-7507-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848404 INFO  (zkCallback-7516-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1848427 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[83603EC0FA962B10]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-4-001 of type PULL
   [junit4]   2> 1848428 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1848428 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1848428 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1848428 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1848513 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1848513 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1848513 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1848524 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a7d603f{/ff_zaj/uu,null,AVAILABLE}
   [junit4]   2> 1848544 INFO  (zkConnectionManagerCallback-7524-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848547 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@d97dca{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43640}
   [junit4]   2> 1848547 INFO  (closeThreadPool-7471-thread-1) [    ] o.e.j.s.Server Started @1848603ms
   [junit4]   2> 1848547 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/tempDir-001/jetty4, replicaType=PULL, solrconfig=solrconfig.xml, hostContext=/ff_zaj/uu, hostPort=43640, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-4-001/cores}
   [junit4]   2> 1848548 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1848549 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1848553 ERROR (closeThreadPool-7471-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1848553 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1848553 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1848553 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1848553 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1848553 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-14T06:30:11.323Z
   [junit4]   2> 1848593 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1848612 INFO  (zkConnectionManagerCallback-7527-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848614 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1848614 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-4-001/solr.xml
   [junit4]   2> 1848618 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1848618 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1848620 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1848650 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848744 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848745 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848747 INFO  (closeThreadPool-7471-thread-3) [n:127.0.0.1:34343_ff_zaj%2Fuu    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-3-001/cores
   [junit4]   2> 1848790 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1848805 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1a6a2717[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1848833 WARN  (closeThreadPool-7471-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@15f7033d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1848834 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42000/solr
   [junit4]   2> 1848837 INFO  (zkConnectionManagerCallback-7533-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848840 INFO  (zkConnectionManagerCallback-7535-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848844 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1848848 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:43640_ff_zaj%2Fuu as DOWN
   [junit4]   2> 1848849 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1848849 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43640_ff_zaj%2Fuu
   [junit4]   2> 1848851 INFO  (zkCallback-7507-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848851 INFO  (zkCallback-7489-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848851 INFO  (zkCallback-7516-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848851 INFO  (zkCallback-7457-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848851 INFO  (zkCallback-7482-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848851 INFO  (zkCallback-7500-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848852 INFO  (zkCallback-7469-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848852 INFO  (zkCallback-7450-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848864 INFO  (zkCallback-7523-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848872 INFO  (zkCallback-7534-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1848879 INFO  (zkConnectionManagerCallback-7542-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1848881 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1848882 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42000/solr ready
   [junit4]   2> 1848882 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1848915 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848932 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848933 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1698bee5
   [junit4]   2> 1848935 INFO  (closeThreadPool-7471-thread-1) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_83603EC0FA962B10-001/shard-4-001/cores
   [junit4]   2> 1848992 DEBUG (ScheduledTrigger-8020-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5 and last live nodes: 3
   [junit4]   2> 1849072 INFO  (qtp1391128725-30804) [n:127.0.0.1:33825_ff_zaj%2Fuu    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36975_ff_zaj%252Fuu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1849092 INFO  (OverseerThreadFactory-8022-thread-3-processing-n:127.0.0.1:33825_ff_zaj%2Fuu) [n:127.0.0.1:33825_ff_zaj%2Fuu c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:36975_ff_zaj%2Fuu for creating new replica of shard shard1 for collection collection1
   [junit4]   2> 1849094 INFO  (OverseerThreadFactory-8022-thread-3-processing-n:127.0.0.1:33825_ff_zaj%2Fuu) [n:127.0.0.1:33825_ff_zaj%2Fuu c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 1849119 INFO  (qtp2141403431-30864) [n:127.0.0.1:36975_ff_zaj%2Fuu    x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1849993 DEBUG (ScheduledTrigger-8020-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5 and last live nodes: 5
   [junit4]   2> 1850156 INFO  (qtp2141403431-30864) [n:127.0.0.1:36975_ff_zaj%2Fuu c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1850205 INFO  (qtp2141403431-30864) [n:127.0.0.1:36975_ff_zaj%2Fuu c:collection1 s:shard1  x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 185041

[...truncated too long message...]

vy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1683544990
     [copy] Copying 236 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1683544990
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1683544990

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 55ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 283 minutes 15 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 20 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/20/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.cdcr.CdcrBootstrapTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)  at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)  at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)   expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)  at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)  at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
        at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
        at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

 expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
        at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
        at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

>
        at __randomizedtesting.SeedInfo.seed([4EC2A3320DFA5CD6]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.failNotNull(Assert.java:755)
        at org.junit.Assert.assertNull(Assert.java:737)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:322)
        at sun.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([4EC2A3320DFA5CD6:7D708BF6004D8661]:0)
        at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:425)
        at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:132)
        at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:116)
        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:288)
        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:368)
        at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83)
        at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:676)
        at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:279)
        at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
        at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:86)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)




Build Log:
[...truncated 13666 lines...]
   [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/init-core-data-001
   [junit4]   2> 447582 WARN  (SUITE-CdcrBootstrapTest-seed#[4EC2A3320DFA5CD6]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 447582 INFO  (SUITE-CdcrBootstrapTest-seed#[4EC2A3320DFA5CD6]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 447616 INFO  (SUITE-CdcrBootstrapTest-seed#[4EC2A3320DFA5CD6]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 447616 INFO  (SUITE-CdcrBootstrapTest-seed#[4EC2A3320DFA5CD6]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4] IGNOR/A 0.01s J2 | CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J2 | CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 447624 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithSourceCluster
   [junit4]   2> 447624 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-target-001
   [junit4]   2> 447625 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 447632 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 447632 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 447732 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer start zk server on port:33376
   [junit4]   2> 447732 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33376
   [junit4]   2> 447732 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33376
   [junit4]   2> 447792 INFO  (zkConnectionManagerCallback-2641-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 447824 INFO  (zkConnectionManagerCallback-2643-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 447837 INFO  (zkConnectionManagerCallback-2645-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 447851 WARN  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 447860 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 447860 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 447860 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 447873 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 447873 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 447874 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 447874 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2592ca99{/solr,null,AVAILABLE}
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5aca4b45{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35743}
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.e.j.s.Server Started @447990ms
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35743}
   [junit4]   2> 447876 ERROR (jetty-launcher-2646-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 447876 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-15T04:02:37.346Z
   [junit4]   2> 447888 INFO  (zkConnectionManagerCallback-2648-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 447889 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 448391 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 448393 WARN  (jetty-launcher-2646-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4a12b22b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 448413 WARN  (jetty-launcher-2646-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@219724c2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 448414 INFO  (jetty-launcher-2646-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33376/solr
   [junit4]   2> 448436 INFO  (zkConnectionManagerCallback-2654-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 448476 INFO  (zkConnectionManagerCallback-2656-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 448590 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35743_solr
   [junit4]   2> 448591 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.c.Overseer Overseer (id=72959193407619077-127.0.0.1:35743_solr-n_0000000000) starting
   [junit4]   2> 448608 INFO  (zkConnectionManagerCallback-2663-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 448611 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33376/solr ready
   [junit4]   2> 448616 INFO  (OverseerStateUpdate-72959193407619077-127.0.0.1:35743_solr-n_0000000000) [n:127.0.0.1:35743_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:35743_solr
   [junit4]   2> 448624 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35743_solr
   [junit4]   2> 448671 INFO  (zkCallback-2655-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 448712 INFO  (zkCallback-2662-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 448756 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 448858 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35743.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 448887 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35743.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 448887 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35743.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 448900 INFO  (jetty-launcher-2646-thread-1) [n:127.0.0.1:35743_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-target-001/node1/.
   [junit4]   2> 448994 INFO  (zkConnectionManagerCallback-2669-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 448996 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 448997 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33376/solr ready
   [junit4]   1> Target zkHost = 127.0.0.1:33376/solr
   [junit4]   2> 448998 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-source-001
   [junit4]   2> 448998 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 448998 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 448998 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 449098 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer start zk server on port:37141
   [junit4]   2> 449098 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:37141
   [junit4]   2> 449099 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 37141
   [junit4]   2> 449272 INFO  (zkConnectionManagerCallback-2671-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449278 INFO  (zkConnectionManagerCallback-2673-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449280 INFO  (zkConnectionManagerCallback-2675-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449283 WARN  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 449296 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 449296 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 449296 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 449305 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 449305 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 449305 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 449306 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@789922ee{/solr,null,AVAILABLE}
   [junit4]   2> 449307 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@73a361bd{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43379}
   [junit4]   2> 449307 INFO  (jetty-launcher-2676-thread-1) [    ] o.e.j.s.Server Started @449421ms
   [junit4]   2> 449307 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43379}
   [junit4]   2> 449307 ERROR (jetty-launcher-2676-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 449307 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 449307 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 449308 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 449308 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 449308 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-15T04:02:38.778Z
   [junit4]   2> 449310 INFO  (zkConnectionManagerCallback-2678-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449311 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 449750 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 449751 WARN  (jetty-launcher-2676-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1663909[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 449755 WARN  (jetty-launcher-2676-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@24fbbbbb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 449757 INFO  (jetty-launcher-2676-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37141/solr
   [junit4]   2> 449758 INFO  (zkConnectionManagerCallback-2684-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449774 INFO  (zkConnectionManagerCallback-2686-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449882 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43379_solr
   [junit4]   2> 449883 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.c.Overseer Overseer (id=72959193495896069-127.0.0.1:43379_solr-n_0000000000) starting
   [junit4]   2> 449924 INFO  (zkConnectionManagerCallback-2693-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 449928 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37141/solr ready
   [junit4]   2> 449929 INFO  (OverseerStateUpdate-72959193495896069-127.0.0.1:43379_solr-n_0000000000) [n:127.0.0.1:43379_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:43379_solr
   [junit4]   2> 449929 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43379_solr
   [junit4]   2> 449940 INFO  (zkCallback-2692-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 449941 INFO  (zkCallback-2685-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 450047 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 450238 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43379.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 450251 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43379.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 450251 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43379.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 450255 INFO  (jetty-launcher-2676-thread-1) [n:127.0.0.1:43379_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-source-001/node1/.
   [junit4]   2> 450355 INFO  (zkConnectionManagerCallback-2699-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 450369 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 450370 INFO  (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[4EC2A3320DFA5CD6]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37141/solr ready
   [junit4]   2> 450378 INFO  (zkConnectionManagerCallback-2701-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 450405 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 450420 INFO  (OverseerThreadFactory-2246-thread-1-processing-n:127.0.0.1:43379_solr) [n:127.0.0.1:43379_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection cdcr-source
   [junit4]   2> 450541 INFO  (OverseerStateUpdate-72959193495896069-127.0.0.1:43379_solr-n_0000000000) [n:127.0.0.1:43379_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"cdcr-source",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"cdcr-source_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:43379/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 450754 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr    x:cdcr-source_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
   [junit4]   2> 450754 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr    x:cdcr-source_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 451794 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 451802 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 451807 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 451810 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 451816 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
   [junit4]   2> 451828 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
   [junit4]   2> 451828 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica_n1' using configuration from collection cdcr-source, trusted=true
   [junit4]   2> 451829 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43379.solr.core.cdcr-source.shard1.replica_n1' (registry 'solr.core.cdcr-source.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d9348f5
   [junit4]   2> 451830 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-source-001/node1/cdcr-source_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBootstrapTest_4EC2A3320DFA5CD6-001/cdcr-source-001/node1/./cdcr-source_shard1_replica_n1/data/]
   [junit4]   2> 452028 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 452028 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 452031 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 452031 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 452034 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4178f0c[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 452036 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
   [junit4]   2> 452036 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
   [junit4]   2> 452036 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
   [junit4]   2> 452037 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 452037 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 452054 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
   [junit4]   2> 452061 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
   [junit4]   2> 452129 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1628042755151233024
   [junit4]   2> 452142 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@4178f0c[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 452146 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-source/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 452146 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/cdcr-source/leaders/shard1
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43379/solr/cdcr-source_shard1_replica_n1/
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43379/solr/cdcr-source_shard1_replica_n1/ has no replicas
   [junit4]   2> 452150 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/cdcr-source/leaders/shard1/leader after winning as /collections/cdcr-source/leader_elect/shard1/election/72959193495896069-core_node2-n_0000000000
   [junit4]   2> 452151 INFO  (zkCallback-2685-thread-1) [    ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
   [junit4]   2> 452154 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43379/solr/cdcr-source_shard1_replica_n1/ shard1
   [junit4]   2> 452256 INFO  (zkCallback-2685-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 452257 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 452260 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1506
   [junit4]   2> 452263 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 452359 INFO  (zkCallback-2685-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 452359 INFO  (zkCallback-2685-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 452360 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=1955
   [junit4]   2> 452422 INFO  (OverseerCollectionConfigSetProcessor-72959193495896069-127.0.0.1:43379_solr-n_0000000000) [n:127.0.0.1:43379_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 452447 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-source/terms/shard1 to Terms{values={core_node2=1}, version=1}
   [junit4]   2> 452448 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=70
   [junit4]   2> 452478 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042755517186048,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 452478 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042755517186048
   [junit4]   2> 452647 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4459f0[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 452649 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@4459f0[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}])))}
   [junit4]   2> 452650 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 452650 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=173
   [junit4]   2> 452701 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=49
   [junit4]   2> 452703 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042755753115648,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 452704 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042755753115648
   [junit4]   2> 452793 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cfa1138[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 452794 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 452796 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@6cfa1138[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}])))}
   [junit4]   2> 452796 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=94
   [junit4]   2> 452831 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=32
   [junit4]   2> 452833 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042755889430528,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 452833 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042755889430528
   [junit4]   2> 452866 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@783a8d86[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 452868 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@783a8d86[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}])))}
   [junit4]   2> 452869 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 452869 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=37
   [junit4]   2> 452900 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=28
   [junit4]   2> 452912 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042755972268032,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 452913 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042755972268032
   [junit4]   2> 452955 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@58aa4239[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 452957 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@58aa4239[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}])))}
   [junit4]   2> 452957 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 452957 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=45
   [junit4]   2> 452974 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=13
   [junit4]   2> 452975 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756038328320,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 452976 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756038328320
   [junit4]   2> 453021 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e6eec5[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453023 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@7e6eec5[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}])))}
   [junit4]   2> 453023 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453023 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=48
   [junit4]   2> 453048 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=16
   [junit4]   2> 453050 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756116971520,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453050 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756116971520
   [junit4]   2> 453331 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@cb79c15[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453333 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@cb79c15[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}])))}
   [junit4]   2> 453333 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453334 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=284
   [junit4]   2> 453383 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=47
   [junit4]   2> 453389 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756472438784,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453389 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756472438784
   [junit4]   2> 453445 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@38621dfd[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453447 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453461 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@38621dfd[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}])))}
   [junit4]   2> 453462 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=73
   [junit4]   2> 453478 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=14
   [junit4]   2> 453480 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756566810624,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453480 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756566810624
   [junit4]   2> 453512 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ef8abff[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453515 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@ef8abff[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}]) Uninverting(_7(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562955}])))}
   [junit4]   2> 453516 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453516 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=37
   [junit4]   2> 453541 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=23
   [junit4]   2> 453543 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756633919488,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453543 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756633919488
   [junit4]   2> 453598 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@11144638[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453600 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@11144638[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}]) Uninverting(_7(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562955}]) Uninverting(_8(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563019}])))}
   [junit4]   2> 453600 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453601 INFO  (qtp193713016-9013) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=58
   [junit4]   2> 453619 INFO  (qtp193713016-9014) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=16
   [junit4]   2> 453621 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756715708416,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453621 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756715708416
   [junit4]   2> 453678 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6177c4df[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453680 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@6177c4df[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}]) Uninverting(_7(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562955}]) Uninverting(_8(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563019}]) Uninverting(_9(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563097}])))}
   [junit4]   2> 453682 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453682 INFO  (qtp193713016-9012) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=62
   [junit4]   2> 453734 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=50
   [junit4]   2> 453738 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756838391808,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453738 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756838391808
   [junit4]   2> 453835 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@19a726d[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453837 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@19a726d[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}]) Uninverting(_7(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562955}]) Uninverting(_8(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563019}]) Uninverting(_9(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563097}]) Uninverting(_a(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563213}])))}
   [junit4]   2> 453840 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 453840 INFO  (qtp193713016-9015) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=105
   [junit4]   2> 453855 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=13
   [junit4]   2> 453878 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1628042756985192448,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 453878 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1b507e3a commitCommandVersion:1628042756985192448
   [junit4]   2> 453972 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@27970324[cdcr-source_shard1_replica_n1] main]
   [junit4]   2> 453974 INFO  (searcherExecutor-2251-thread-1-processing-n:127.0.0.1:43379_solr x:cdcr-source_shard1_replica_n1 c:cdcr-source s:shard1 r:core_node2) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@27970324[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562052}]) Uninverting(_1(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562194}]) Uninverting(_2(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562313}]) Uninverting(_3(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562396}]) Uninverting(_4(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562451}]) Uninverting(_5(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562528}]) Uninverting(_6(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562871}]) Uninverting(_7(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622562955}]) Uninverting(_8(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563019}]) Uninverting(_9(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563097}]) Uninverting(_a(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563213}]) Uninverting(_b(8.0.0):C100:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=8.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552622563397}])))}
   [junit4]   2> 453975 INFO  (qtp193713016-9011) [n:127.0.0.1:43379_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler

[...truncated too long message...]

llapseTest, AlternateDirectoryTest, LeaderVoteWaitTimeoutTest, ZookeeperStatusHandlerTest, SolrTestCaseJ4Test, TestSimpleTrackingShardHandler, JavabinLoaderTest, TestAuthenticationFramework, PreAnalyzedUpdateProcessorTest, ConfigSetsAPITest, TestSolrCloudWithKerberosAlt, TestSolrIndexConfig, CloudMLTQParserTest, Tagger2Test, OutputWriterTest, TestSizeLimitedDistributedMap, TestMaxTokenLenTokenizer, DocValuesNotIndexedTest, ShardsWhitelistTest, TestFieldCacheSortRandom, CdcrBootstrapTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrBootstrapTest -Dtests.seed=4EC2A3320DFA5CD6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/test-data/enwiki.random.lines.txt -Dtests.locale=sr -Dtests.timezone=America/Knox_IN -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | CdcrBootstrapTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
   [junit4]    > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
   [junit4]    > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
   [junit4]    > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
   [junit4]    > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
   [junit4]    > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
   [junit4]    > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    >  expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
   [junit4]    > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
   [junit4]    > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
   [junit4]    > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
   [junit4]    > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
   [junit4]    > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
   [junit4]    > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]    > >
   [junit4]    > at __randomizedtesting.SeedInfo.seed([4EC2A3320DFA5CD6]:0)
   [junit4]    > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:322)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [145/839 (1!)] on J2 in 92.00s, 3 tests, 1 failure, 2 skipped <<< FAILURES!

[...truncated 118 lines...]
   [junit4] Suite: org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrdsUninvertLimit -Dtests.method=testTriggerUnInvertLimit -Dtests.seed=4EC2A3320DFA5CD6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-KW -Dtests.timezone=America/Boa_Vista -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   99.8s J1 | TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > at __randomizedtesting.SeedInfo.seed([4EC2A3320DFA5CD6:7D708BF6004D8661]:0)
   [junit4]    > at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:425)
   [junit4]    > at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:132)
   [junit4]    > at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:116)
   [junit4]    > at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:288)
   [junit4]    > at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:368)
   [junit4]    > at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113)
   [junit4]    > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83)
   [junit4]    > at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:676)
   [junit4]    > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:279)
   [junit4]    > at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
   [junit4]    > at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:86)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene80): {field=PostingsFormat(name=Direct)}, docValues:{}, maxPointsInLeafNode=576, maxMBSortInHeap=5.879346143370903, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@26958b1b), locale=ar-KW, timezone=America/Boa_Vista
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=413679432,total=455606272
   [junit4]   2> NOTE: All tests run in this JVM: [TestSearcherReuse, TestApiFramework, TestCoreAdminApis, SchemaApiFailureTest, TaggingAttributeTest, HdfsDirectoryTest, ExternalFileFieldSortTest, IndexSizeTriggerTest, DateFieldTest, CloudExitableDirectoryReaderTest, ResponseHeaderTest, TestValueSourceCache, TestSchemaVersionResource, TestManagedSchema, TestShortCircuitedRequests, TestBM25SimilarityFactory, OpenExchangeRatesOrgProviderTest, BlockCacheTest, UninvertDocValuesMergePolicyTest, TestHashPartitioner, BJQParserTest, TestHttpShardHandlerFactory, TestSimpleTextCodec, TestConfigSetsAPI, TestCloudDeleteByQuery, CoreAdminOperationTest, RulesTest, TlogReplayBufferedWhileIndexingTest, TestSchemaNameResource, ExecutePlanActionTest, BasicDistributedZk2Test, TestSimPolicyCloud, TestCollectionsAPIViaSolrCloudCluster, BasicFunctionalityTest, TestEmbeddedSolrServerConstructors, TestDynamicURP, DefaultValueUpdateProcessorTest, TestOmitPositions, TestFieldCacheWithThreads, CdcrBidirectionalTest, DeleteShardTest, HdfsNNFailoverTest, TestRestManager, HttpPartitionTest, TestLuceneMatchVersion, ClassificationUpdateProcessorIntegrationTest, TokenizerChainTest, ZkStateWriterTest, FullSolrCloudDistribCmdsTest, SpellCheckCollatorTest, TestCloudPivotFacet, SpatialRPTFieldTypeTest, TestCollationField, ExitableDirectoryReaderTest, TestXIncludeConfig, DOMUtilTest, ReplaceNodeNoTargetTest, MinimalSchemaTest, TestSimExecutePlanAction, TestDocTermOrdsUninvertLimit]
   [junit4] Completed [183/839 (2!)] on J1 in 99.90s, 1 test, 1 error <<< FAILURES!

[...truncated 2237 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/temp/junit4-J1-20190315_035509_2726606370985192163549.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/heapdumps/java_pid2728.hprof ...
   [junit4] Heap dump file created [479304380 bytes in 2.852 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 8947 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid2728.hprof

Total time: 293 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 21 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/21/

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, SolrCore] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:305)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)   expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, SolrCore] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:502)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:305)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)  at java.lang.Thread.run(Thread.java:748)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, SolrCore]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
        at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:305)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, SolrCore]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
        at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
        at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1056)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:876)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:502)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
        at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:305)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)

>
        at __randomizedtesting.SeedInfo.seed([DBE4744D9E021A1A]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.failNotNull(Assert.java:755)
        at org.junit.Assert.assertNull(Assert.java:737)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:322)
        at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([DBE4744D9E021A1A:53B04B9730FE77E2]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
        at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
        at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([DBE4744D9E021A1A:50C3A79CDF04B19E]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
        at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
        at __randomizedtesting.SeedInfo.seed([DBE4744D9E021A1A:53B04B9730FE77E2]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:143)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:138)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1032)
        at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:164)
        at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:108)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14580 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/init-core-data-001
   [junit4]   2> 1248006 INFO  (SUITE-RollingRestartTest-seed#[DBE4744D9E021A1A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1248008 INFO  (SUITE-RollingRestartTest-seed#[DBE4744D9E021A1A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1248009 INFO  (SUITE-RollingRestartTest-seed#[DBE4744D9E021A1A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1248009 INFO  (SUITE-RollingRestartTest-seed#[DBE4744D9E021A1A]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1248021 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1248022 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1248022 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1248122 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer start zk server on port:34304
   [junit4]   2> 1248122 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34304
   [junit4]   2> 1248122 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34304
   [junit4]   2> 1248128 INFO  (zkConnectionManagerCallback-6860-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248132 INFO  (zkConnectionManagerCallback-6862-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248133 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1248135 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1248136 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1248137 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1248138 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1248140 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1248141 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1248142 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1248143 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1248144 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1248145 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1248148 INFO  (zkConnectionManagerCallback-6866-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248151 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1248342 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1248343 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1248343 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1248343 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1248344 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1248344 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1248344 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1248345 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5bcd1dbc{/,null,AVAILABLE}
   [junit4]   2> 1248345 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@33eb7a71[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1248346 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7773339b{SSL,[ssl, http/1.1]}{127.0.0.1:43959}
   [junit4]   2> 1248346 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.s.Server Started @1248400ms
   [junit4]   2> 1248346 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=43959, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/control-001/cores}
   [junit4]   2> 1248346 ERROR (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1248346 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1248346 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1248347 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1248347 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1248347 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-16T01:07:42.048Z
   [junit4]   2> 1248348 INFO  (zkConnectionManagerCallback-6868-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248350 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1248350 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/control-001/solr.xml
   [junit4]   2> 1248355 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1248355 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1248357 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1248477 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1248478 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1248479 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@51737b69[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1248483 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1248484 WARN  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@736e12f6[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1248485 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34304/solr
   [junit4]   2> 1248486 INFO  (zkConnectionManagerCallback-6874-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248489 INFO  (zkConnectionManagerCallback-6876-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248595 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43959_
   [junit4]   2> 1248596 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.c.Overseer Overseer (id=72964167794032645-127.0.0.1:43959_-n_0000000000) starting
   [junit4]   2> 1248603 INFO  (zkConnectionManagerCallback-6883-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248605 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34304/solr ready
   [junit4]   2> 1248606 INFO  (OverseerStateUpdate-72964167794032645-127.0.0.1:43959_-n_0000000000) [n:127.0.0.1:43959_    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:43959_
   [junit4]   2> 1248607 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43959_
   [junit4]   2> 1248622 INFO  (zkCallback-6875-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248624 INFO  (zkCallback-6882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248625 DEBUG (OverseerAutoScalingTriggerThread-72964167794032645-127.0.0.1:43959_-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:43959_]
   [junit4]   2> 1248628 DEBUG (ScheduledTrigger-6458-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1248638 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1248664 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1248679 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1248680 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1248681 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [n:127.0.0.1:43959_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/control-001/cores
   [junit4]   2> 1248710 INFO  (zkConnectionManagerCallback-6889-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248712 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248713 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34304/solr ready
   [junit4]   2> 1248731 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43959_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1248735 INFO  (OverseerThreadFactory-6460-thread-1-processing-n:127.0.0.1:43959_) [n:127.0.0.1:43959_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1248846 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1248847 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1249631 DEBUG (ScheduledTrigger-6458-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1249869 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1249886 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1250020 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1250039 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1250039 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1250040 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1250043 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1250054 WARN  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1250123 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1250123 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1250126 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1250126 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1250127 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7375675330393155]
   [junit4]   2> 1250128 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@59a9de83[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1250129 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1250130 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1250131 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1250131 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1628122344411103232
   [junit4]   2> 1250134 INFO  (searcherExecutor-6465-thread-1-processing-n:127.0.0.1:43959_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@59a9de83[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1250137 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1250137 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1250141 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1250141 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1250141 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43959/control_collection_shard1_replica_n1/
   [junit4]   2> 1250141 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1250142 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:43959/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1250142 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72964167794032645-core_node2-n_0000000000
   [junit4]   2> 1250144 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43959/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1250246 INFO  (zkCallback-6875-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1250246 INFO  (zkCallback-6875-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1250247 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1250250 INFO  (qtp1621945413-28904) [n:127.0.0.1:43959_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1404
   [junit4]   2> 1250253 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1250350 INFO  (zkCallback-6875-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1250350 INFO  (zkCallback-6875-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1250350 INFO  (zkCallback-6875-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1250351 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43959_&wt=javabin&version=2} status=0 QTime=1619
   [junit4]   2> 1250357 INFO  (zkConnectionManagerCallback-6895-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1250358 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1250359 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34304/solr ready
   [junit4]   2> 1250359 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1250362 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1250368 INFO  (OverseerThreadFactory-6460-thread-2-processing-n:127.0.0.1:43959_) [n:127.0.0.1:43959_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1250369 INFO  (OverseerCollectionConfigSetProcessor-72964167794032645-127.0.0.1:43959_-n_0000000000) [n:127.0.0.1:43959_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1250577 WARN  (OverseerThreadFactory-6460-thread-2-processing-n:127.0.0.1:43959_) [n:127.0.0.1:43959_    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1250580 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1250581 INFO  (qtp1621945413-28906) [n:127.0.0.1:43959_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=219
   [junit4]   2> 1250582 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=16
   [junit4]   2> 1250631 DEBUG (ScheduledTrigger-6458-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
   [junit4]   2> 1250751 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-1-001 of type NRT
   [junit4]   2> 1250753 WARN  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1250754 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1250754 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1250754 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1250756 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1250756 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1250756 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1250756 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e3d3fca{/,null,AVAILABLE}
   [junit4]   2> 1250757 WARN  (closeThreadPool-6896-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@53c81e14[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@671a4e40{SSL,[ssl, http/1.1]}{127.0.0.1:35861}
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.e.j.s.Server Started @1250812ms
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=35861, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-1-001/cores}
   [junit4]   2> 1250758 ERROR (closeThreadPool-6896-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1250758 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1250759 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-16T01:07:44.460Z
   [junit4]   2> 1250761 INFO  (zkConnectionManagerCallback-6898-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1250762 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1250762 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-1-001/solr.xml
   [junit4]   2> 1250768 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1250768 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1250770 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1250928 INFO  (TEST-RollingRestartTest.test-seed#[DBE4744D9E021A1A]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-2-001 of type NRT
   [junit4]   2> 1250938 WARN  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1250941 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1250941 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1250941 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1250943 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1250943 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1250943 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1250945 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@13830882{/,null,AVAILABLE}
   [junit4]   2> 1250946 WARN  (closeThreadPool-6896-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@753f766b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1250946 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@18959d79{SSL,[ssl, http/1.1]}{127.0.0.1:38635}
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.e.j.s.Server Started @1251001ms
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=38635, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-2-001/cores}
   [junit4]   2> 1250947 ERROR (closeThreadPool-6896-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1250947 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-16T01:07:44.648Z
   [junit4]   2> 1250950 INFO  (zkConnectionManagerCallback-6900-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1250951 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1250951 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_DBE4744D9E021A1A-001/shard-2-001/solr.xml
   [junit4]   2> 1250954 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1250954 WARN  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1250955 WARN  (closeThreadPool-6896-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6218961e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1250956 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1250956 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1250958 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1250959 WARN  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1250960 WARN  (closeThreadPool-6896-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4e1c99e1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1250961 INFO  (closeThreadPool-6896-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34304/solr
   [junit4]   2> 1250962 INFO  (zkConnectionManagerCallback-6906-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1250965 INFO  (zkConnectionManagerCallback-6908-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1250971 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1250979 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.ZkController Publish node=127.0.0.1:35861_ as DOWN
   [junit4]   2> 1250980 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1250981 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35861_
   [junit4]   2> 1250982 INFO  (zkCallback-6882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1250982 INFO  (zkCallback-6894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1250982 INFO  (zkCallback-6875-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1250983 INFO  (zkCallback-6907-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1251014 INFO  (zkConnectionManagerCallback-6915-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1251016 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1251017 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34304/solr ready
   [junit4]   2> 1251017 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1251025 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1251026 WARN  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1251027 WARN  (closeThreadPool-6896-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@119c460c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1251031 WARN  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1251031 WARN  (closeThreadPool-6896-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@586949ad[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1251033 INFO  (closeThreadPool-6896-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34304/solr
   [junit4]   2> 1251035 INFO  (zkConnectionManagerCallback-6922-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1251038 INFO  (zkConnectionManagerCallback-6924-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1251044 INFO  (closeThreadPool-6896-thread-2) [n:127.0.0.1:38635_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1251048 INFO  (closeThreadPool-6896-thread-2) [n:127.0.0.1:38635_    ] o.a.s.c.ZkController Publish node=127.0.0.1:38635_ as DOWN
   [junit4]   2> 1251050 INFO  (closeThreadPool-6896-thread-2) [n:127.0.0.1:38635_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1251050 INFO  (closeThreadPool-6896-thread-2) [n:127.0.0.1:38635_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38635_
   [junit4]   2> 1251051 INFO  (zkCallback-6894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251051 INFO  (zkCallback-6907-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251052 INFO  (zkCallback-6875-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251052 INFO  (zkCallback-6882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251052 INFO  (zkCallback-6914-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251057 INFO  (zkCallback-6923-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1251057 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1251074 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1251075 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76d1b539
   [junit4]   2> 1251076 INFO  (closeThreadPool-6896-thread-1) [n:127.0.0.1:35861_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/j

[...truncated too long message...]

lve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null45721093
     [copy] Copying 236 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null45721093
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null45721093

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 54ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 318 minutes 8 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 22 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/22/

2 tests failed.
FAILED:  org.apache.solr.handler.export.TestExportWriter.testStringWithCase

Error Message:
expected:<...:"3"},       {"id":"[1"},       {"id":"4]"},       {"id":"2"}...> but was:<...:"3"},       {"id":"[4"},       {"id":"1]"},       {"id":"2"}...>

Stack Trace:
org.junit.ComparisonFailure: expected:<...:"3"},
      {"id":"[1"},
      {"id":"4]"},
      {"id":"2"}...> but was:<...:"3"},
      {"id":"[4"},
      {"id":"1]"},
      {"id":"2"}...>
        at __randomizedtesting.SeedInfo.seed([8D7B0E38E250331:6C9C4C6517A80354]:0)
        at org.junit.Assert.assertEquals(Assert.java:115)
        at org.junit.Assert.assertEquals(Assert.java:144)
        at org.apache.solr.handler.export.TestExportWriter.assertJsonEquals(TestExportWriter.java:497)
        at org.apache.solr.handler.export.TestExportWriter.testStringWithCase(TestExportWriter.java:349)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([8D7B0E38E250331:80838F3920D96EC9]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
        at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
        at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:53)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14116 lines...]
   [junit4] Suite: org.apache.solr.cloud.RollingRestartTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/init-core-data-001
   [junit4]   2> 1172827 WARN  (SUITE-RollingRestartTest-seed#[8D7B0E38E250331]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1172828 INFO  (SUITE-RollingRestartTest-seed#[8D7B0E38E250331]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1172831 INFO  (SUITE-RollingRestartTest-seed#[8D7B0E38E250331]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1172831 INFO  (SUITE-RollingRestartTest-seed#[8D7B0E38E250331]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1172831 INFO  (SUITE-RollingRestartTest-seed#[8D7B0E38E250331]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /o_oz/h
   [junit4]   2> 1172835 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1172836 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1172836 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1172936 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer start zk server on port:41936
   [junit4]   2> 1172936 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:41936
   [junit4]   2> 1172936 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 41936
   [junit4]   2> 1172945 INFO  (zkConnectionManagerCallback-6274-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1172949 INFO  (zkConnectionManagerCallback-6276-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1172950 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1172951 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1172953 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172954 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1172955 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1172956 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1172957 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1172958 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1172960 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1172961 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1172962 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1172965 INFO  (zkConnectionManagerCallback-6280-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1172968 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1173216 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1173216 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1173216 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1173216 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1173221 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1173221 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1173221 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1173226 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@12283b5b{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1173226 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6102561d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5af18d30{SSL,[ssl, http/1.1]}{127.0.0.1:32935}
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.s.Server Started @1173286ms
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/control/data, hostContext=/o_oz/h, hostPort=32935, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/control-001/cores}
   [junit4]   2> 1173228 ERROR (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1173228 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:44.858Z
   [junit4]   2> 1173234 INFO  (zkConnectionManagerCallback-6282-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1173236 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1173236 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/control-001/solr.xml
   [junit4]   2> 1173244 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1173244 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1173248 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1173615 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1173615 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1173616 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2eac5820[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1173620 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1173620 WARN  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4352dae0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1173622 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1173624 INFO  (zkConnectionManagerCallback-6288-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1173626 INFO  (zkConnectionManagerCallback-6290-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1173724 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32935_o_oz%2Fh
   [junit4]   2> 1173725 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.Overseer Overseer (id=72970589165322245-127.0.0.1:32935_o_oz%2Fh-n_0000000000) starting
   [junit4]   2> 1173733 INFO  (zkConnectionManagerCallback-6297-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1173735 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1173737 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32935_o_oz%2Fh
   [junit4]   2> 1173737 INFO  (OverseerStateUpdate-72970589165322245-127.0.0.1:32935_o_oz%2Fh-n_0000000000) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:32935_o_oz%2Fh
   [junit4]   2> 1173742 INFO  (zkCallback-6296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1173750 INFO  (OverseerStateUpdate-72970589165322245-127.0.0.1:32935_o_oz%2Fh-n_0000000000) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1173771 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1173800 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1173816 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1173817 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1173819 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/control-001/cores
   [junit4]   2> 1173851 INFO  (zkConnectionManagerCallback-6303-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1173852 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1173854 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1173868 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32935_o_oz%252Fh&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1173871 INFO  (OverseerThreadFactory-5205-thread-1-processing-n:127.0.0.1:32935_o_oz%2Fh) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1173982 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1173982 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1175014 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1175046 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1175236 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1175264 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1175264 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1175265 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1175269 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=70.984375, floorSegmentMB=0.8046875, forceMergeDeletesPctAllowed=21.6805790754208, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=21.440169439091413
   [junit4]   2> 1175282 WARN  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1175348 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1175348 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1175364 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1175364 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1175366 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=507232357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1175367 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6782c14a[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1175368 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1175369 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1175369 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1175369 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1628225087208423424
   [junit4]   2> 1175374 INFO  (searcherExecutor-5210-thread-1-processing-n:127.0.0.1:32935_o_oz%2Fh x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6782c14a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1175376 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1175376 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1175380 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1175380 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1175380 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:32935/o_oz/h/control_collection_shard1_replica_n1/
   [junit4]   2> 1175381 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1175381 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:32935/o_oz/h/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1175381 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72970589165322245-core_node2-n_0000000000
   [junit4]   2> 1175383 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:32935/o_oz/h/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1175485 INFO  (zkCallback-6289-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1175487 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1175489 INFO  (qtp1968449705-24760) [n:127.0.0.1:32935_o_oz%2Fh c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1507
   [junit4]   2> 1175499 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1175589 INFO  (zkCallback-6289-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1175589 INFO  (zkCallback-6289-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1175590 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32935_o_oz%252Fh&wt=javabin&version=2} status=0 QTime=1722
   [junit4]   2> 1175599 INFO  (zkConnectionManagerCallback-6309-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1175600 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1175602 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1175602 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1175606 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1175611 INFO  (OverseerThreadFactory-5205-thread-2-processing-n:127.0.0.1:32935_o_oz%2Fh) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1175611 INFO  (OverseerCollectionConfigSetProcessor-72970589165322245-127.0.0.1:32935_o_oz%2Fh-n_0000000000) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1175819 WARN  (OverseerThreadFactory-5205-thread-2-processing-n:127.0.0.1:32935_o_oz%2Fh) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1175822 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1175823 INFO  (qtp1968449705-24761) [n:127.0.0.1:32935_o_oz%2Fh    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=217
   [junit4]   2> 1175824 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=16
   [junit4]   2> 1176013 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-1-001 of type NRT
   [junit4]   2> 1176016 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1176016 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1176016 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1176016 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1176017 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1176017 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1176018 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1176018 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@496d4c33{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1176018 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@34bbbb2e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176019 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@14bd62bf{SSL,[ssl, http/1.1]}{127.0.0.1:44507}
   [junit4]   2> 1176019 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server Started @1176077ms
   [junit4]   2> 1176019 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=44507, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-1-001/cores}
   [junit4]   2> 1176019 ERROR (closeThreadPool-6310-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1176020 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1176020 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1176020 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1176020 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1176020 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:47.650Z
   [junit4]   2> 1176022 INFO  (zkConnectionManagerCallback-6312-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176023 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1176023 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-1-001/solr.xml
   [junit4]   2> 1176028 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1176028 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1176031 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1176131 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1176131 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176132 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@56d7d489[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176135 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176136 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5da6c5f5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176138 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1176145 INFO  (zkConnectionManagerCallback-6318-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176150 INFO  (zkConnectionManagerCallback-6320-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176155 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1176159 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.ZkController Publish node=127.0.0.1:44507_o_oz%2Fh as DOWN
   [junit4]   2> 1176160 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1176160 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44507_o_oz%2Fh
   [junit4]   2> 1176162 INFO  (zkCallback-6296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1176162 INFO  (zkCallback-6289-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1176162 INFO  (zkCallback-6319-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1176167 INFO  (zkCallback-6308-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1176193 INFO  (zkConnectionManagerCallback-6327-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176194 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1176195 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1176196 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1176205 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-2-001 of type NRT
   [junit4]   2> 1176207 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1176207 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1176207 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1176208 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1176209 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1176209 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1176209 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1176209 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@28651ab3{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1176210 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@58a14cec[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@724fdef2{SSL,[ssl, http/1.1]}{127.0.0.1:37117}
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server Started @1176269ms
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=37117, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-2-001/cores}
   [junit4]   2> 1176211 ERROR (closeThreadPool-6310-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1176211 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:47.841Z
   [junit4]   2> 1176214 INFO  (zkConnectionManagerCallback-6330-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176215 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1176215 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-2-001/solr.xml
   [junit4]   2> 1176221 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1176221 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1176223 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1176227 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176245 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176245 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176247 INFO  (closeThreadPool-6310-thread-1) [n:127.0.0.1:44507_o_oz%2Fh    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-1-001/cores
   [junit4]   2> 1176273 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1176274 WARN  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176279 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5809cc3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176283 WARN  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176284 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1ecd686f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176285 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1176287 INFO  (zkConnectionManagerCallback-6336-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176290 INFO  (zkConnectionManagerCallback-6338-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176295 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1176300 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.ZkController Publish node=127.0.0.1:37117_o_oz%2Fh as DOWN
   [junit4]   2> 1176301 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1176301 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37117_o_oz%2Fh
   [junit4]   2> 1176302 INFO  (zkCallback-6308-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176302 INFO  (zkCallback-6319-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176302 INFO  (zkCallback-6289-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176302 INFO  (zkCallback-6296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176303 INFO  (zkCallback-6337-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176309 INFO  (zkCallback-6326-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1176333 INFO  (zkConnectionManagerCallback-6345-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176335 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1176336 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1176337 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1176369 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176412 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176412 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176414 INFO  (closeThreadPool-6310-thread-2) [n:127.0.0.1:37117_o_oz%2Fh    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-2-001/cores
   [junit4]   2> 1176435 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-3-001 of type NRT
   [junit4]   2> 1176437 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1176438 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1176438 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1176438 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1176448 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1176448 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1176448 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1176449 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d7f6a71{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1176449 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@34ca2f6d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@476bcf5b{SSL,[ssl, http/1.1]}{127.0.0.1:39458}
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server Started @1176508ms
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=39458, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-3-001/cores}
   [junit4]   2> 1176450 ERROR (closeThreadPool-6310-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1176450 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:48.080Z
   [junit4]   2> 1176453 INFO  (zkConnectionManagerCallback-6348-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176454 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1176454 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-3-001/solr.xml
   [junit4]   2> 1176460 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1176460 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1176462 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1176607 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1176607 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176608 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@56cbd09f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176612 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176613 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4958f8ca[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176614 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1176616 INFO  (zkConnectionManagerCallback-6354-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176618 INFO  (zkConnectionManagerCallback-6356-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176624 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1176628 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:39458_o_oz%2Fh as DOWN
   [junit4]   2> 1176629 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1176629 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39458_o_oz%2Fh
   [junit4]   2> 1176631 INFO  (zkCallback-6319-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6289-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6308-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6326-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6355-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176631 INFO  (zkCallback-6337-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176632 INFO  (zkCallback-6344-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1176639 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-4-001 of type NRT
   [junit4]   2> 1176641 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1176641 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1176641 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1176641 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1176643 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1176643 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1176643 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1176643 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6333358c{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1176644 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6ee0b8a2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176644 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@547770c4{SSL,[ssl, http/1.1]}{127.0.0.1:44326}
   [junit4]   2> 1176644 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server Started @1176702ms
   [junit4]   2> 1176644 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=44326, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-4-001/cores}
   [junit4]   2> 1176653 ERROR (closeThreadPool-6310-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1176653 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1176653 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1176653 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1176653 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1176653 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:48.283Z
   [junit4]   2> 1176655 INFO  (zkConnectionManagerCallback-6360-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176656 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1176656 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-4-001/solr.xml
   [junit4]   2> 1176662 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1176662 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1176663 INFO  (zkConnectionManagerCallback-6365-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176664 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1176664 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1176665 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1176666 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1176699 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176717 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176717 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176719 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-3-001/cores
   [junit4]   2> 1176739 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1176740 WARN  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176741 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4debf38a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176745 WARN  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1176745 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@67c4ac61[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176747 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1176748 INFO  (zkConnectionManagerCallback-6372-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176751 INFO  (zkConnectionManagerCallback-6374-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176757 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1176764 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:44326_o_oz%2Fh as DOWN
   [junit4]   2> 1176765 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1176765 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44326_o_oz%2Fh
   [junit4]   2> 1176767 INFO  (zkCallback-6319-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176767 INFO  (zkCallback-6326-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6308-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6337-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6344-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6355-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6289-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176768 INFO  (zkCallback-6364-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176769 INFO  (zkCallback-6373-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1176819 INFO  (zkConnectionManagerCallback-6381-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176821 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1176823 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
   [junit4]   2> 1176823 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1176853 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176856 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-5-001 of type NRT
   [junit4]   2> 1176858 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1176858 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1176858 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1176858 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1176860 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1176860 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1176860 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1176860 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@468bb8f7{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1176861 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5538536d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1176861 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@282655{SSL,[ssl, http/1.1]}{127.0.0.1:40961}
   [junit4]   2> 1176861 INFO  (closeThreadPool-6310-thread-1) [    ] o.e.j.s.Server Started @1176920ms
   [junit4]   2> 1176861 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty5, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=40961, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-5-001/cores}
   [junit4]   2> 1176862 ERROR (closeThreadPool-6310-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1176862 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1176862 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1176862 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1176862 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1176862 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:48.492Z
   [junit4]   2> 1176864 INFO  (zkConnectionManagerCallback-6384-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1176865 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1176865 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-5-001/solr.xml
   [junit4]   2> 1176871 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1176871 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1176871 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176872 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa
   [junit4]   2> 1176873 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1176874 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-4-001/cores
   [junit4]   2> 1177049 INFO  (TEST-RollingRestartTest.test-seed#[8D7B0E38E250331]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 6 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-6-001 of type NRT
   [junit4]   2> 1177051 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1177054 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1177054 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1177054 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1177055 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1177055 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1177055 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1177056 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e5fa569{/o_oz/h,null,AVAILABLE}
   [junit4]   2> 1177057 WARN  (closeThreadPool-6310-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@c8fdeb0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1177057 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1989703a{SSL,[ssl, http/1.1]}{127.0.0.1:32929}
   [junit4]   2> 1177057 INFO  (closeThreadPool-6310-thread-2) [    ] o.e.j.s.Server Started @1177116ms
   [junit4]   2> 1177057 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/tempDir-001/jetty6, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/o_oz/h, hostPort=32929, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-6-001/cores}
   [junit4]   2> 1177058 ERROR (closeThreadPool-6310-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1177058 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1177058 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1177058 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1177058 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1177058 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T04:20:48.688Z
   [junit4]   2> 1177061 INFO  (zkConnectionManagerCallback-6386-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1177063 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1177063 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RollingRestartTest_8D7B0E38E250331-001/shard-6-001/solr.xml
   [junit4]   2> 1177069 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1177069 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1177071 INFO  (closeThreadPool-6310-thread-2) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6a6ce2fa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1177125 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1177126 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1177127 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@498654e6[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1177130 WARN  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 1177131 WARN  (closeThreadPool-6310-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@cf0102e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1177132 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
   [junit4]   2> 1177140 INFO  (zkConnectionManagerCallback-6392-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1177142 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1033e5d3e40001b, likely client has closed socket
   [junit4]   2> 1177144 INFO  (zkConnectionManagerCallback-6394-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1177150 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1177155 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:40961_o_oz%2Fh as DOWN
   [junit4]   2> 1177156 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1177156 INFO  (closeThreadPool-6310-thread-1) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40961_o_oz%2Fh
   [junit4]   2> 1177159 INFO  (zkCallback-6337-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1177159 INFO  (zkCallback-6289-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1177159 INFO  (zkCallback-6355-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1177159 INFO  (zkCallback-6308-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1177159 INFO  (zkCallback-6373-thread-1) [  

[...truncated too long message...]



ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null706151056
     [copy] Copying 236 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null706151056
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null706151056

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 93ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 270 minutes 30 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 23 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/23/

3 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at __randomizedtesting.SeedInfo.seed([F3A57687EE8439A5:7882A556AF829221]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
        at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=15790, name=qtp1543806316-15790-acceptor-0@42ed0701-ServerConnector@681793c6{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45877}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=15789, name=qtp1543806316-15789, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=15796, name=Scheduler-1306611292, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=15854, name=Scheduler-466240204, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=15792, name=qtp1543806316-15792, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=15795, name=qtp1543806316-15795, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=15788, name=qtp1543806316-15788, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=15793, name=qtp1543806316-15793, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=15791, name=qtp1543806316-15791, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=15794, name=qtp1543806316-15794, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
   1) Thread[id=15790, name=qtp1543806316-15790-acceptor-0@42ed0701-ServerConnector@681793c6{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45877}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=15789, name=qtp1543806316-15789, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=15796, name=Scheduler-1306611292, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=15854, name=Scheduler-466240204, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=15792, name=qtp1543806316-15792, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=15795, name=qtp1543806316-15795, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=15788, name=qtp1543806316-15788, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=15793, name=qtp1543806316-15793, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=15791, name=qtp1543806316-15791, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=15794, name=qtp1543806316-15794, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([F3A57687EE8439A5]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=15789, name=qtp1543806316-15789, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=15796, name=Scheduler-1306611292, state=TIMED_WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=15854, name=Scheduler-466240204, state=WAITING, group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=15788, name=qtp1543806316-15788, state=RUNNABLE, group=TGRP-ShardSplitTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=15789, name=qtp1543806316-15789, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=15796, name=Scheduler-1306611292, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=15854, name=Scheduler-466240204, state=WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=15788, name=qtp1543806316-15788, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$42/1748553791.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([F3A57687EE8439A5]:0)




Build Log:
[...truncated 14412 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.ShardSplitTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/init-core-data-001
   [junit4]   2> 1123851 WARN  (SUITE-ShardSplitTest-seed#[F3A57687EE8439A5]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=72 numCloses=72
   [junit4]   2> 1123851 INFO  (SUITE-ShardSplitTest-seed#[F3A57687EE8439A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1123853 INFO  (SUITE-ShardSplitTest-seed#[F3A57687EE8439A5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1123853 INFO  (SUITE-ShardSplitTest-seed#[F3A57687EE8439A5]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1123853 INFO  (SUITE-ShardSplitTest-seed#[F3A57687EE8439A5]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1123857 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1123858 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1123858 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1123958 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer start zk server on port:34372
   [junit4]   2> 1123958 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34372
   [junit4]   2> 1123958 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34372
   [junit4]   2> 1123962 INFO  (zkConnectionManagerCallback-3930-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1123965 INFO  (zkConnectionManagerCallback-3932-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1123966 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1123968 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1123969 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1123970 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1123971 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1123972 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1123974 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1123975 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1123976 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1123977 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1123978 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1123979 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 1124156 WARN  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1124156 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1124156 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1124156 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1124157 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1124157 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1124157 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1124160 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e92d358{/,null,AVAILABLE}
   [junit4]   2> 1124160 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@49f08355{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:40628}
   [junit4]   2> 1124160 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.s.Server Started @1124223ms
   [junit4]   2> 1124160 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=40628, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/control-001/cores}
   [junit4]   2> 1124161 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1124161 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1124161 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1124161 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1124161 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1124161 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T23:24:51.606Z
   [junit4]   2> 1124163 INFO  (zkConnectionManagerCallback-3934-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1124164 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1124164 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/control-001/solr.xml
   [junit4]   2> 1124184 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1124184 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1124186 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@606586a1, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1124341 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1124343 WARN  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@347fe819[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1124348 WARN  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1b6ef760[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1124349 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34372/solr
   [junit4]   2> 1124351 INFO  (zkConnectionManagerCallback-3940-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1124354 INFO  (zkConnectionManagerCallback-3942-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1124453 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40628_
   [junit4]   2> 1124454 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer Overseer (id=72975088004431876-127.0.0.1:40628_-n_0000000000) starting
   [junit4]   2> 1124460 INFO  (zkConnectionManagerCallback-3949-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1124463 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34372/solr ready
   [junit4]   2> 1124464 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 1124464 INFO  (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:40628_
   [junit4]   2> 1124465 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40628_
   [junit4]   2> 1124466 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0 #Completed tasks: 0
   [junit4]   2> 1124467 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 1124467 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1124467 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 1124467 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 1124467 INFO  (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 1124474 INFO  (zkCallback-3948-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1124494 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1124522 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@606586a1
   [junit4]   2> 1124535 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@606586a1
   [junit4]   2> 1124535 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@606586a1
   [junit4]   2> 1124537 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [n:127.0.0.1:40628_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/control-001/cores
   [junit4]   2> 1124567 INFO  (zkConnectionManagerCallback-3955-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1124568 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1124569 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34372/solr ready
   [junit4]   2> 1124571 INFO  (qtp1591648398-12042) [n:127.0.0.1:40628_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40628_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1124574 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
   [junit4]   2> 1124574 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
   [junit4]   2> 1124574 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:40628_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 1124575 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
   [junit4]   2> 1124575 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:40628_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 1124575 INFO  (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 1124575 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 0
   [junit4]   2> 1124575 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 1124575 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 1124575 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 1124575 INFO  (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 1124576 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/control_collection
   [junit4]   2> 1124576 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:control_collection
   [junit4]   2> 1124577 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 1124579 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:40628_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 1124579 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 1124580 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/control_collection/state.json
   [junit4]   2> 1124680 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection control_collection, shardNames [shard1] , message : {
   [junit4]   2>   "name":"control_collection",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "createNodeSet":"127.0.0.1:40628_",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 1124682 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Creating core control_collection_shard1_replica_n1 as part of shard shard1 of collection control_collection on 127.0.0.1:40628_
   [junit4]   2> 1124684 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1124685 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1124688 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:40628",
   [junit4]   2>   "node_name":"127.0.0.1:40628_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 1124688 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:40628",
   [junit4]   2>   "node_name":"127.0.0.1:40628_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 1124689 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:40628","node_name":"127.0.0.1:40628_","state":"down","type":"NRT"}
   [junit4]   2> 1124689 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/0)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:40628",
   [junit4]   2>           "node_name":"127.0.0.1:40628_",
   [junit4]   2>           "state":"down",
   [junit4]   2>           "type":"NRT"}}}}}
   [junit4]   2> 1124790 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 0
   [junit4]   2> 1125706 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1125723 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1125830 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1125848 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 1125849 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@606586a1
   [junit4]   2> 1125849 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 1125853 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=57.591796875, floorSegmentMB=1.654296875, forceMergeDeletesPctAllowed=2.3611854592723027, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.527042965043741, deletesPctAllowed=46.570279818070716
   [junit4]   2> 1125862 WARN  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1125922 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1125922 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1125924 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1125924 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1125926 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.3837890625, noCFSRatio=1.0]
   [junit4]   2> 1125927 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4a13fcce[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1125928 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1125928 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1125929 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1125929 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1628297068149735424
   [junit4]   2> 1125933 INFO  (searcherExecutor-3499-thread-1-processing-n:127.0.0.1:40628_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4a13fcce[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1125936 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 1125936 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 1125939 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1125939 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1125939 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40628/control_collection_shard1_replica_n1/
   [junit4]   2> 1125940 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1125940 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:40628/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1125940 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72975088004431876-core_node2-n_0000000000
   [junit4]   2> 1125942 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40628/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1125943 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:40628",
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "state":"active"} current state version: 0
   [junit4]   2> 1126043 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 1
   [junit4]   2> 1126044 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126044 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126045 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1126047 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:40628",
   [junit4]   2>   "node_name":"127.0.0.1:40628_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 1126047 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "core_node_name":"core_node2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:40628",
   [junit4]   2>   "node_name":"127.0.0.1:40628_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 1126047 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:40628","node_name":"127.0.0.1:40628_","state":"active","type":"NRT","leader":"true"}
   [junit4]   2> 1126047 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/2)={
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{"core_node2":{
   [junit4]   2>           "core":"control_collection_shard1_replica_n1",
   [junit4]   2>           "base_url":"http://127.0.0.1:40628",
   [junit4]   2>           "node_name":"127.0.0.1:40628_",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT",
   [junit4]   2>           "leader":"true"}}}}}
   [junit4]   2> 1126049 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1365
   [junit4]   2> 1126049 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Finished create command on all shards for collection: control_collection
   [junit4]   2> 1126050 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 1126054 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
   [junit4]   2> 1126054 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 1126054 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 1126054 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 1126054 INFO  (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 1126054 DEBUG (OverseerThreadFactory-3494-thread-1-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{success={127.0.0.1:40628_={responseHeader={status=0,QTime=1365},core=control_collection_shard1_replica_n1}}}
   [junit4]   2> 1126056 INFO  (qtp1591648398-12042) [n:127.0.0.1:40628_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1126148 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 2
   [junit4]   2> 1126149 INFO  (zkCallback-3941-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126149 INFO  (zkCallback-3941-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126149 INFO  (zkCallback-3941-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126150 INFO  (qtp1591648398-12042) [n:127.0.0.1:40628_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40628_&wt=javabin&version=2} status=0 QTime=1578
   [junit4]   2> 1126156 INFO  (zkConnectionManagerCallback-3961-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1126157 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1126159 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34372/solr ready
   [junit4]   2> 1126159 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1126160 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1126163 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
   [junit4]   2> 1126163 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
   [junit4]   2> 1126163 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"1",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 1126163 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
   [junit4]   2> 1126164 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 1
   [junit4]   2> 1126164 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"1",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 1126164 INFO  (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1126164 INFO  (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1126165 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1
   [junit4]   2> 1126165 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 1126165 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 1126165 DEBUG (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 1126165 INFO  (OverseerCollectionConfigSetProcessor-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 1126165 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:collection1
   [junit4]   2> 1126166 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 1126168 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "createNodeSet":"",
   [junit4]   2>   "stateFormat":"1",
   [junit4]   2>   "nrtReplicas":"1",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 1126168 DEBUG (OverseerStateUpdate-72975088004431876-127.0.0.1:40628_-n_0000000000) [n:127.0.0.1:40628_    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 1126368 WARN  (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1126369 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.a.c.CreateCollectionCmd Finished create command for collection: collection1
   [junit4]   2> 1126369 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 1126371 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as completed.
   [junit4]   2> 1126371 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 1126371 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 1126371 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 1126371 INFO  (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 1126371 DEBUG (OverseerThreadFactory-3494-thread-2-processing-n:127.0.0.1:40628_) [n:127.0.0.1:40628_    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000002 complete, response:{}
   [junit4]   2> 1126372 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 1126373 INFO  (qtp1591648398-12044) [n:127.0.0.1:40628_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=212
   [junit4]   2> 1126374 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=4
   [junit4]   2> 1126544 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit2-seed#[F3A57687EE8439A5]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/shard-1-001 of type NRT
   [junit4]   2> 1126545 WARN  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 1126545 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 1126545 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1126545 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 1126550 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1126551 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1126551 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1126554 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3bb90ca3{/,null,AVAILABLE}
   [junit4]   2> 1126554 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@256723fa{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:42244}
   [junit4]   2> 1126554 INFO  (closeThreadPool-3962-thread-1) [    ] o.e.j.s.Server Started @1126617ms
   [junit4]   2> 1126554 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=42244, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.ShardSplitTest_F3A57687EE8439A5-001/shard-1-001/cores}
   [junit4]   2> 1126555 ERROR (closeThreadPool-3962-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1126555 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1126555 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1126555 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1126555 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1126555 INFO  (closeThreadPool-3962-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-03-17T23:24:54Z
   [junit4]   2> 1126559 INFO  (zkConnectionManagerCallbac

[...truncated too long message...]

-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1268994173
     [copy] Copying 236 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1268994173
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1268994173

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 152ms :: artifacts dl 21ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 231 minutes 48 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-8.0 - Build # 24 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.0/24/

1 tests failed.
FAILED:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings

Error Message:
last stage: inconsistent endOffset at pos=1: 31 vs 36; token=rxivgWTeeqeRkxivgudsrmkftkxivgs ut 1

Stack Trace:
java.lang.IllegalStateException: last stage: inconsistent endOffset at pos=1: 31 vs 36; token=rxivgWTeeqeRkxivgudsrmkftkxivgs ut 1
        at __randomizedtesting.SeedInfo.seed([920FA2A878492625:F8541DB9210706D6]:0)
        at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:146)
        at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:746)
        at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:657)
        at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:559)
        at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:899)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 4067 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> stage 0: r<[0-1] +1> WTeeqeRk<[11-19] +1> udsrmkftk<[20-29] +1> s<[30-31] +1> ut<[32-34] +1> 1<[35-36] +1> ң<[37-38] +1> drfxmm<[39-45] +1> ߄߄<[51-52] +1> 🀋<[53-55] +1> ek<[56-58] +1>
   [junit4]   2> stage 1: r WTeeqeRk udsrmkftk s ut 1 ң drfxmm ߄߄ 🀋 ek<[0-58] +0> WTeeqeRk<[11-19] +1> WTeeqeRk udsrmkftk s<[11-31] +0> WTeeqeRk udsrmkftk s ut<[11-34] +0> WTeeqeRk udsrmkftk s ut 1<[11-36] +0> WTeeqeRk udsrmkftk s ut 1 ң<[11-38] +0> WTeeqeRk udsrmkftk s ut 1 ң drfxmm<[11-45] +0> WTeeqeRk udsrmkftk s ut 1 ң drfxmm ߄߄<[11-52] +0> WTeeqeRk udsrmkftk s ut 1 ң drfxmm ߄߄ 🀋<[11-55] +0> WTeeqeRk udsrmkftk s ut 1 ң drfxmm ߄߄ 🀋 ek<[11-58] +0> udsrmkftk<[20-29] +1> udsrmkftk s ut<[20-34] +0> udsrmkftk s ut 1<[20-36] +0> udsrmkftk s ut 1 ң<[20-38] +0> udsrmkftk s ut 1 ң drfxmm<[20-45] +0> udsrmkftk s ut 1 ң drfxmm ߄߄<[20-52] +0> udsrmkftk s ut 1 ң drfxmm ߄߄ 🀋<[20-55] +0> udsrmkftk s ut 1 ң drfxmm ߄߄ 🀋 ek<[20-58] +0> s<[30-31] +1> s ut 1<[30-36] +0>
   [junit4]   2> last stage: rxivgWTeeqeRkxivgudsrmkftkxivgs<[0-31] +1> rxivgWTeeqeRkxivgudsrmkftkxivgs ut 1<[0-36] +0>
   [junit4]   2> TEST FAIL: useCharFilter=true text='r \u20a2\u20ba\u20b0\u20bf\u20cc\u20bc\u20a3\u20c6 WTeeqeRk udsrmkftk s ut 1$\u04a3 drfxmm &<  \ufbba\u07c4 \ud83c\udc0b ek'
   [junit4]   2> Exception from random analyzer:
   [junit4]   2> charfilters=
   [junit4]   2>   org.apache.lucene.analysis.pattern.PatternReplaceCharFilter(a, <HANGUL>, java.io.StringReader@79cd111)
   [junit4]   2>   org.apache.lucene.analysis.charfilter.HTMLStripCharFilter(org.apache.lucene.analysis.pattern.PatternReplaceCharFilter@6629b22a)
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer(org.apache.lucene.util.AttributeFactory$1@1046af7d)
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@2e2249c0 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1,flags=0,payload=null, 3, 14)
   [junit4]   2>   org.apache.lucene.analysis.shingle.FixedShingleFilter(ValidatingTokenFilter@50aac5f7 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1,flags=0,payload=null, 4, xivg, <ALPHANUM>)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChainsWithLargeStrings -Dtests.seed=920FA2A878492625 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-EG -Dtests.timezone=Europe/Riga -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   5.07s J1 | TestRandomChains.testRandomChainsWithLargeStrings <<<
   [junit4]    > Throwable #1: java.lang.IllegalStateException: last stage: inconsistent endOffset at pos=1: 31 vs 36; token=rxivgWTeeqeRkxivgudsrmkftkxivgs ut 1
   [junit4]    > at __randomizedtesting.SeedInfo.seed([920FA2A878492625:F8541DB9210706D6]:0)
   [junit4]    > at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:146)
   [junit4]    > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:746)
   [junit4]    > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:657)
   [junit4]    > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:559)
   [junit4]    > at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:899)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/build/analysis/common/test/J1/temp/lucene.analysis.core.TestRandomChains_920FA2A878492625-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene80): {dummy=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1061, maxMBSortInHeap=7.6297934768298346, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6cadabd3), locale=ar-EG, timezone=Europe/Riga
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=131447128,total=302514176
   [junit4]   2> NOTE: All tests run in this JVM: [TestCJKWidthFilterFactory, TestScandinavianFoldingFilter, TestConcatenateGraphFilterFactory, TestFixBrokenOffsetsFilter, TestElisionFilterFactory, TestAsciiFoldingFilterFactory, TestArabicFilters, TestGermanAnalyzer, TypeAsPayloadTokenFilterTest, TestPersianNormalizationFilterFactory, TestGreekStemmer, TestPatternReplaceFilterFactory, TestIrishAnalyzer, TestCodepointCountFilter, TestSynonymFilterFactory, TestHindiFilters, TestCompoundWordTokenFilter, TestTurkishLowerCaseFilter, TestSpanishLightStemFilter, NGramTokenFilterTest, TestDutchAnalyzer, TestUnicodeWhitespaceTokenizer, TestRussianAnalyzer, TestFinnishLightStemFilterFactory, TestScandinavianNormalizationFilterFactory, TestBengaliStemmer, TestLimitTokenPositionFilterFactory, TestItalianLightStemFilter, TestPortugueseLightStemFilterFactory, FixedShingleFilterTest, TestSerbianNormalizationFilterFactory, TestEnglishMinimalStemFilterFactory, Test64kAffixes, TestNorwegianLightStemFilterFactory, TestAllAnalyzersHaveFactories, TestLimitTokenOffsetFilter, TestFactories, TestHomonyms, TestHungarianAnalyzer, CommonGramsFilterTest, TestMorphAlias, TestGermanMinimalStemFilter, TestUAX29URLEmailTokenizerFactory, GreekAnalyzerTest, TestGalicianStemFilter, TestApostropheFilter, TestLithuanianStemming, DateRecognizerFilterTest, TestOnlyInCompound, TestArabicStemFilter, TestBulgarianAnalyzer, TestBulgarianStemFilterFactory, TestBulgarianStemmer, TestBengaliAnalyzer, TestBengaliFilters, TestBrazilianStemFilterFactory, TestCatalanAnalyzer, HTMLStripCharFilterTest, TestHTMLStripCharFilterFactory, TestMappingCharFilter, TestCJKBigramFilter, TestDecimalDigitFilter, TestDecimalDigitFilterFactory, TestKeywordAnalyzer, TestKeywordTokenizer, TestRandomChains]
   [junit4] Completed [248/291 (1!)] on J1 in 305.68s, 2 tests, 1 error <<< FAILURES!

[...truncated 61438 lines...]
[asciidoctor:convert] asciidoctor: ERROR: about-this-guide.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
[asciidoctor:convert] asciidoctor: ERROR: solr-glossary.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
     [java] Processed 2491 links (2044 relative) to 3300 anchors in 251 files
     [echo] Validated Links & Anchors via: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/build/solr-ref-guide/bare-bones-html/

-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec]
     [exec] Crawl/parse...
     [exec]
     [exec] Verify...
     [echo] Checking for malformed docs...

jar-checksums:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

common.resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/null146571619
     [copy] Copying 36 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/null146571619
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/null146571619

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1279785731
     [copy] Copying 236 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1279785731
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.0/checkout/solr/null1279785731

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 82ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 571 minutes 14 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
        at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
                at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1072)
                at hudson.FilePath.act(FilePath.java:1061)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
                at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1835)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath.act(FilePath.java:1074)
        at hudson.FilePath.act(FilePath.java:1061)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1835)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]