[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 55 - Still Failing

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
10 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 55 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/55/

11 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
6 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:     1) Thread[id=142598, name=zkCallback-34457-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=142597, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    4) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    6) Thread[id=142547, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
   1) Thread[id=142598, name=zkCallback-34457-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=142597, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   4) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   6) Thread[id=142547, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=142598, name=zkCallback-34457-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=142598, name=zkCallback-34457-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
KeeperErrorCode = Session expired for /overseer_elect/leader

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:48F0D554D9C33217]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderId(OverseerTaskProcessor.java:374)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderNode(OverseerTaskProcessor.java:365)
        at org.apache.solr.cloud.RollingRestartTest.waitUntilOverseerDesignateIsLeader(RollingRestartTest.java:137)
        at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
        at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:39E979214B4A1265]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:48F0D554D9C33217]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:108)
        at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
        at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:90)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 44 more


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:48F0D554D9C33217]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.UnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=22655, name=testExecutor-5153-thread-11, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=22655, name=testExecutor-5153-thread-11, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35890/gm/sq
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35890/gm/sq
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
        ... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        ... 9 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:44358

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44358
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:48F0D554D9C33217]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:315)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        ... 43 more


FAILED:  org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth

Error Message:
No match for authorization/permissions[1]/collection = x, full response = {   "responseHeader":{     "status":0,     "QTime":0},   "authorization.enabled":true,   "authorization":{     "class":"solr.RuleBasedAuthorizationPlugin",     "user-role":{       "solr":"admin",       "harry":"admin"},     "permissions":[{         "name":"security-edit",         "role":"admin"}],     "":{"v":3}}}

Stack Trace:
java.lang.AssertionError: No match for authorization/permissions[1]/collection = x, full response = {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "authorization.enabled":true,
  "authorization":{
    "class":"solr.RuleBasedAuthorizationPlugin",
    "user-role":{
      "solr":"admin",
      "harry":"admin"},
    "permissions":[{
        "name":"security-edit",
        "role":"admin"}],
    "":{"v":3}}}

        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:7CCA9C9CD36CDC95]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.security.BasicAuthIntegrationTest.verifySecurityStatus(BasicAuthIntegrationTest.java:283)
        at org.apache.solr.security.BasicAuthIntegrationTest.verifySecurityStatus(BasicAuthIntegrationTest.java:248)
        at org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth(BasicAuthIntegrationTest.java:159)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF:48F0D554D9C33217]:0)
        at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=24170, name=Thread-11041, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=24170, name=Thread-11041, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50344/sc_z/jn/collection2_shard3_replica_n12: Cannot talk to ZooKeeper - Updates are disabled.
        at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:549)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:633)




Build Log:
[...truncated 12893 lines...]
   [junit4] Suite: org.apache.solr.security.BasicAuthIntegrationTest
   [junit4]   2> 0    INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/init-core-data-001
   [junit4]   2> 2149 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2654 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.e.j.u.log Logging initialized @23801ms
   [junit4]   2> 2713 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 6110 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001
   [junit4]   2> 6175 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 6209 INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6210 INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 9190 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:38902
   [junit4]   2> 9992 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 10432 ERROR (Thread-1) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 14552 INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 14592 INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 14608 INFO  (jetty-launcher-1-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 16122 INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a7f4c4d{/solr,null,AVAILABLE}
   [junit4]   2> 16264 INFO  (jetty-launcher-1-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2744c2a2{/solr,null,AVAILABLE}
   [junit4]   2> 16290 INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@121fe42a{/solr,null,AVAILABLE}
   [junit4]   2> 16663 INFO  (jetty-launcher-1-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7f096b7f{SSL,[ssl, http/1.1]}{127.0.0.1:44995}
   [junit4]   2> 16664 INFO  (jetty-launcher-1-thread-2) [    ] o.e.j.s.Server Started @37812ms
   [junit4]   2> 16664 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44995}
   [junit4]   2> 16760 INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7c3a2191{SSL,[ssl, http/1.1]}{127.0.0.1:32902}
   [junit4]   2> 16760 INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.Server Started @37909ms
   [junit4]   2> 16685 INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@459bd5c5{SSL,[ssl, http/1.1]}{127.0.0.1:49549}
   [junit4]   2> 16773 INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.Server Started @37921ms
   [junit4]   2> 16760 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=32902}
   [junit4]   2> 16773 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49549}
   [junit4]   2> 17022 ERROR (jetty-launcher-1-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 17022 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 17040 ERROR (jetty-launcher-1-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 17044 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 17072 ERROR (jetty-launcher-1-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 17072 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 17088 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 17088 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 17107 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 17108 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 17108 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 17108 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 17225 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T14:33:34.071Z
   [junit4]   2> 17226 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T14:33:34.044Z
   [junit4]   2> 17235 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T14:33:34.025Z
   [junit4]   2> 17780 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 18088 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 18298 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 18522 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 22152 INFO  (jetty-launcher-1-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38902/solr
   [junit4]   2> 22199 INFO  (jetty-launcher-1-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38902/solr
   [junit4]   2> 22200 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38902/solr
   [junit4]   2> 25659 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 25676 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32902_solr
   [junit4]   2> 25943 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.c.Overseer Overseer (id=98759790185611272-127.0.0.1:32902_solr-n_0000000000) starting
   [junit4]   2> 26281 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 26543 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44995_solr
   [junit4]   2> 26550 INFO  (zkCallback-17-thread-1-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 26568 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 26580 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 26588 INFO  (zkCallback-16-thread-1-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 26615 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49549_solr
   [junit4]   2> 26676 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 26677 INFO  (zkCallback-17-thread-1-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 26677 INFO  (zkCallback-16-thread-1-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 29486 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32902_solr
   [junit4]   2> 29488 INFO  (zkCallback-16-thread-1-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 29488 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 29563 INFO  (OverseerStateUpdate-98759790185611272-127.0.0.1:32902_solr-n_0000000000) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 34313 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49549.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34342 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49549.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34343 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49549.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34577 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32902.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34616 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32902.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34616 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32902.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34632 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44995.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34632 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44995.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34632 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44995.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 34779 INFO  (jetty-launcher-1-thread-2) [n:127.0.0.1:44995_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node2/.
   [junit4]   2> 34801 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:32902_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node3/.
   [junit4]   2> 34946 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:49549_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node1/.
   [junit4]   2> 40316 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 40317 INFO  (SUITE-BasicAuthIntegrationTest-seed#[C0A4EA8E773F5FEF]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38902/solr ready
   [junit4]   2> 45776 INFO  (qtp414322899-29) [n:127.0.0.1:32902_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=authCollection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 46272 INFO  (OverseerThreadFactory-14-thread-1-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.CreateCollectionCmd Create collection authCollection
   [junit4]   2> 46665 INFO  (OverseerStateUpdate-98759790185611272-127.0.0.1:32902_solr-n_0000000000) [n:127.0.0.1:32902_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"authCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:32902/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 46701 INFO  (OverseerStateUpdate-98759790185611272-127.0.0.1:32902_solr-n_0000000000) [n:127.0.0.1:32902_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"authCollection_shard2_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:44995/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 46720 INFO  (OverseerStateUpdate-98759790185611272-127.0.0.1:32902_solr-n_0000000000) [n:127.0.0.1:32902_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"authCollection_shard3_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:49549/solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 47606 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=authCollection_shard1_replica_n1&action=CREATE&numShards=3&collection=authCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 47620 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=authCollection_shard2_replica_n3&action=CREATE&numShards=3&collection=authCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 47700 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 47703 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 47841 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=authCollection_shard3_replica_n5&action=CREATE&numShards=3&collection=authCollection&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 47842 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 48131 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48131 INFO  (zkCallback-16-thread-1-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48132 INFO  (zkCallback-17-thread-1-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48146 INFO  (zkCallback-17-thread-2-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48156 INFO  (zkCallback-16-thread-2-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48360 INFO  (zkCallback-17-thread-2-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48360 INFO  (zkCallback-17-thread-3-processing-n:127.0.0.1:32902_solr) [n:127.0.0.1:32902_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48360 INFO  (zkCallback-16-thread-2-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48361 INFO  (zkCallback-16-thread-1-processing-n:127.0.0.1:44995_solr) [n:127.0.0.1:44995_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48391 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 48924 INFO  (zkCallback-18-thread-2-processing-n:127.0.0.1:49549_solr) [n:127.0.0.1:49549_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 50666 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 50695 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 50696 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 52091 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema [authCollection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 52098 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.s.IndexSchema [authCollection_shard3_replica_n5] Schema name=minimal
   [junit4]   2> 52114 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.s.IndexSchema [authCollection_shard2_replica_n3] Schema name=minimal
   [junit4]   2> 52377 WARN  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 52387 WARN  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 52419 WARN  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 52388 WARN  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 52435 WARN  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 52450 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 52451 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard1_replica_n1' using configuration from collection authCollection, trusted=true
   [junit4]   2> 52590 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 52591 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard2_replica_n3' using configuration from collection authCollection, trusted=true
   [junit4]   2> 52598 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 52598 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard3_replica_n5' using configuration from collection authCollection, trusted=true
   [junit4]   2> 52675 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44995.solr.core.authCollection.shard2.replica_n3' (registry 'solr.core.authCollection.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 52693 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49549.solr.core.authCollection.shard3.replica_n5' (registry 'solr.core.authCollection.shard3.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 52695 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32902.solr.core.authCollection.shard1.replica_n1' (registry 'solr.core.authCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7024682a
   [junit4]   2> 52716 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 52733 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 52732 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 52762 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.c.SolrCore [[authCollection_shard3_replica_n5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node1/authCollection_shard3_replica_n5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node1/./authCollection_shard3_replica_n5/data/]
   [junit4]   2> 52762 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore [[authCollection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node3/authCollection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node3/./authCollection_shard1_replica_n1/data/]
   [junit4]   2> 52762 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.c.SolrCore [[authCollection_shard2_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node2/authCollection_shard2_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.BasicAuthIntegrationTest_C0A4EA8E773F5FEF-001/tempDir-001/node2/./authCollection_shard2_replica_n3/data/]
   [junit4]   2> 66501 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 66501 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 66699 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 66700 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 66722 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 66722 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 66727 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 66740 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 66729 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 66744 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 66764 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 66764 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 66892 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@9bcb4ee[authCollection_shard2_replica_n3] main]
   [junit4]   2> 66894 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@147ad54c[authCollection_shard1_replica_n1] main]
   [junit4]   2> 66893 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@e9321ec[authCollection_shard3_replica_n5] main]
   [junit4]   2> 67265 INFO  (qtp296840392-37) [n:127.0.0.1:49549_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 67267 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 67296 INFO  (qtp1289883220-48) [n:127.0.0.1:44995_solr c:authCollection s:shard2 r:core_node4 x:authCollection_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 67388 INFO  (qtp414322899-32) [n:127.0.0.1:32902_solr c:authCollection s:shard1 r:core_node2 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 6741

[...truncated too long message...]

    at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=142597, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   [junit4]    >    4) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    6) Thread[id=142547, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=142598, name=zkCallback-34457-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=142716, name=zkCallback-34457-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=142717, name=zkCallback-34457-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=142596, name=StoppableCommitThread-SendThread(127.0.0.1:56707), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([C0A4EA8E773F5FEF]:0)
   [junit4] Completed [502/733 (9!)] on J0 in 259.19s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 37938 lines...]
-documentation-lint:
     [echo] checking for broken html...
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec]
     [exec] Crawl/parse...
     [exec]
     [exec] Verify...
     [exec]
     [exec] file:///build/docs/sandbox/org/apache/lucene/document/FloatPointNearestNeighbor.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec]
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:834: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:101: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build.xml:142: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build.xml:155: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:2570: exec returned: 1

Total time: 1007 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 56 - Still Failing

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/56/

9 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413:1BACBC761F66C9EB]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:108)
        at org.apache.solr.cloud.ChaosMonkey.logCollectionStateSummary(ChaosMonkey.java:709)
        at org.apache.solr.cloud.ChaosMonkey.wait(ChaosMonkey.java:703)
        at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:220)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
12 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:     1) Thread[id=6221, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.util.WeakHashMap.get(WeakHashMap.java:403)         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:319)         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:92)         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:114)         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)         at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)    2) Thread[id=6210, name=StoppableSearchThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1217)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)         at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)    3) Thread[id=6331, name=zkCallback-875-thread-3-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=6222, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=6455, name=StoppableCommitThread-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    6) Thread[id=6215, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)         at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)    7) Thread[id=6207, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    8) Thread[id=6332, name=zkCallback-875-thread-3-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    9) Thread[id=6208, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)   10) Thread[id=5894, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=6209, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)   12) Thread[id=6456, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
   1) Thread[id=6221, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.util.WeakHashMap.get(WeakHashMap.java:403)
        at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:319)
        at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:92)
        at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:114)
        at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
        at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)
   2) Thread[id=6210, name=StoppableSearchThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1217)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
   3) Thread[id=6331, name=zkCallback-875-thread-3-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=6222, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=6455, name=StoppableCommitThread-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   6) Thread[id=6215, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
        at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
   7) Thread[id=6207, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   8) Thread[id=6332, name=zkCallback-875-thread-3-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   9) Thread[id=6208, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
  10) Thread[id=5894, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=6209, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
  12) Thread[id=6456, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=6221, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Object.hashCode(Native Method)         at java.util.WeakHashMap.hash(WeakHashMap.java:298)         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)         at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)    2) Thread[id=6210, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.io.PrintStream.write(PrintStream.java:523)         at java.io.PrintStream.print(PrintStream.java:669)         at java.io.PrintStream.println(PrintStream.java:806)         at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:56)    3) Thread[id=6331, name=zkCallback-875-thread-3-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    4) Thread[id=6207, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.io.PrintStream.println(PrintStream.java:805)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)    5) Thread[id=6208, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.io.PrintStream.println(PrintStream.java:805)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)    6) Thread[id=6455, name=StoppableCommitThread-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    7) Thread[id=6209, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at java.io.PrintStream.println(PrintStream.java:805)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=6221, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Object.hashCode(Native Method)
        at java.util.WeakHashMap.hash(WeakHashMap.java:298)
        at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
        at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
        at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)
        at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
        at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
        at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)
   2) Thread[id=6210, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.PrintStream.write(PrintStream.java:523)
        at java.io.PrintStream.print(PrintStream.java:669)
        at java.io.PrintStream.println(PrintStream.java:806)
        at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:56)
   3) Thread[id=6331, name=zkCallback-875-thread-3-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   4) Thread[id=6207, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.PrintStream.println(PrintStream.java:805)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
   5) Thread[id=6208, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.PrintStream.println(PrintStream.java:805)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
   6) Thread[id=6455, name=StoppableCommitThread-SendThread(127.0.0.1:53172), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   7) Thread[id=6209, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.PrintStream.println(PrintStream.java:805)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413:E70862E9F5BEE39C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:108)
        at org.apache.solr.common.cloud.ClusterStateUtil.waitForAllActiveAndLiveReplicas(ClusterStateUtil.java:70)
        at org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest(TestCloudRecovery.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 44 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.UnloadDistributedZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [MDCAwareThreadPoolExecutor] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:872)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1024)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:534)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [MDCAwareThreadPoolExecutor]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:872)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1024)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
        at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:294)
        at sun.reflect.GeneratedMethodAccessor108.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1507121150531,generation=2,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24e.cfe, _24e.cfs, _24e.si, _24f.fdt, _24f.fdx, _24f.fnm, _24f.nvd, _24f.nvm, _24f.si, _24f_Asserting_0.doc, _24f_Asserting_0.tim, _24f_Asserting_0.tip, _24g.fdt, _24g.fdx, _24g.fnm, _24g.nvd, _24g.nvm, _24g.si, _24g_Asserting_0.doc, _24g_Asserting_0.tim, _24g_Asserting_0.tip, _24h.fdt, _24h.fdx, _24h.fnm, _24h.nvd, _24h.nvm, _24h.si, _24h_Asserting_0.doc, _24h_Asserting_0.tim, _24h_Asserting_0.tip, _24i.fdt, _24i.fdx, _24i.fnm, _24i.nvd, _24i.nvm, _24i.si, _24i_Asserting_0.doc, _24i_Asserting_0.tim, _24i_Asserting_0.tip, _24j.fdt, _24j.fdx, _24j.fnm, _24j.nvd, _24j.nvm, _24j.si, _24j_Asserting_0.doc, _24j_Asserting_0.tim, _24j_Asserting_0.tip, _ui.cfe, _ui.cfs, _ui.si, segments_2]}]> but was:<[{indexVersion=1507121150531,generation=2,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24e.cfe, _24e.cfs, _24e.si, _24f.fdt, _24f.fdx, _24f.fnm, _24f.nvd, _24f.nvm, _24f.si, _24f_Asserting_0.doc, _24f_Asserting_0.tim, _24f_Asserting_0.tip, _24g.fdt, _24g.fdx, _24g.fnm, _24g.nvd, _24g.nvm, _24g.si, _24g_Asserting_0.doc, _24g_Asserting_0.tim, _24g_Asserting_0.tip, _24h.fdt, _24h.fdx, _24h.fnm, _24h.nvd, _24h.nvm, _24h.si, _24h_Asserting_0.doc, _24h_Asserting_0.tim, _24h_Asserting_0.tip, _24i.fdt, _24i.fdx, _24i.fnm, _24i.nvd, _24i.nvm, _24i.si, _24i_Asserting_0.doc, _24i_Asserting_0.tim, _24i_Asserting_0.tip, _24j.fdt, _24j.fdx, _24j.fnm, _24j.nvd, _24j.nvm, _24j.si, _24j_Asserting_0.doc, _24j_Asserting_0.tim, _24j_Asserting_0.tip, _ui.cfe, _ui.cfs, _ui.si, segments_2]}, {indexVersion=1507121150531,generation=3,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24k.cfe, _24k.cfs, _24k.si, _ui.cfe, _ui.cfs, _ui.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1507121150531,generation=2,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24e.cfe, _24e.cfs, _24e.si, _24f.fdt, _24f.fdx, _24f.fnm, _24f.nvd, _24f.nvm, _24f.si, _24f_Asserting_0.doc, _24f_Asserting_0.tim, _24f_Asserting_0.tip, _24g.fdt, _24g.fdx, _24g.fnm, _24g.nvd, _24g.nvm, _24g.si, _24g_Asserting_0.doc, _24g_Asserting_0.tim, _24g_Asserting_0.tip, _24h.fdt, _24h.fdx, _24h.fnm, _24h.nvd, _24h.nvm, _24h.si, _24h_Asserting_0.doc, _24h_Asserting_0.tim, _24h_Asserting_0.tip, _24i.fdt, _24i.fdx, _24i.fnm, _24i.nvd, _24i.nvm, _24i.si, _24i_Asserting_0.doc, _24i_Asserting_0.tim, _24i_Asserting_0.tip, _24j.fdt, _24j.fdx, _24j.fnm, _24j.nvd, _24j.nvm, _24j.si, _24j_Asserting_0.doc, _24j_Asserting_0.tim, _24j_Asserting_0.tip, _ui.cfe, _ui.cfs, _ui.si, segments_2]}]> but was:<[{indexVersion=1507121150531,generation=2,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24e.cfe, _24e.cfs, _24e.si, _24f.fdt, _24f.fdx, _24f.fnm, _24f.nvd, _24f.nvm, _24f.si, _24f_Asserting_0.doc, _24f_Asserting_0.tim, _24f_Asserting_0.tip, _24g.fdt, _24g.fdx, _24g.fnm, _24g.nvd, _24g.nvm, _24g.si, _24g_Asserting_0.doc, _24g_Asserting_0.tim, _24g_Asserting_0.tip, _24h.fdt, _24h.fdx, _24h.fnm, _24h.nvd, _24h.nvm, _24h.si, _24h_Asserting_0.doc, _24h_Asserting_0.tim, _24h_Asserting_0.tip, _24i.fdt, _24i.fdx, _24i.fnm, _24i.nvd, _24i.nvm, _24i.si, _24i_Asserting_0.doc, _24i_Asserting_0.tim, _24i_Asserting_0.tip, _24j.fdt, _24j.fdx, _24j.fnm, _24j.nvd, _24j.nvm, _24j.si, _24j_Asserting_0.doc, _24j_Asserting_0.tim, _24j_Asserting_0.tip, _ui.cfe, _ui.cfs, _ui.si, segments_2]}, {indexVersion=1507121150531,generation=3,filelist=[_10v.cfe, _10v.cfs, _10v.si, _17e.cfe, _17e.cfs, _17e.si, _1k4.cfe, _1k4.cfs, _1k4.si, _1ka.cfe, _1ka.cfs, _1ka.si, _1qn.cfe, _1qn.cfs, _1qn.si, _1x0.cfe, _1x0.cfs, _1x0.si, _23j.cfe, _23j.cfs, _23j.si, _24k.cfe, _24k.cfs, _24k.si, _ui.cfe, _ui.cfs, _ui.si, segments_3]}]>
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413:B62F989CC1D2AA10]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.UnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=5627, name=testExecutor-1980-thread-11, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=5627, name=testExecutor-1980-thread-11, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413:1BACBC761F66C9EB]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:52851
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:52851
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
        ... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        ... 9 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=13589, name=testExecutor-2279-thread-17, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=13589, name=testExecutor-2279-thread-17, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33552
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33552
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
        ... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        ... 9 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<4>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<4>
        at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413:1BACBC761F66C9EB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12966 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> 638732 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/init-core-data-001
   [junit4]   2> 638735 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=267 numCloses=267
   [junit4]   2> 638735 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 638777 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 638777 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 648201 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 655436 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 656997 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 657141 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 660120 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 660509 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_45416_hdfs____.x5jnku/webapp
   [junit4]   2> 664862 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45416
   [junit4]   2> 682703 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 682706 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 684046 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_43750_datanode____.lj5mcm/webapp
   [junit4]   2> 689857 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43750
   [junit4]   2> 694104 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 694116 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 694277 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40125_datanode____.2wtyap/webapp
   [junit4]   2> 701754 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40125
   [junit4]   2> 710765 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:37663) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 711129 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:37663) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 715343 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1a2f461e809dc3: from storage DS-6a655397-f967-4ddb-8113-40d611b6dddb node DatanodeRegistration(127.0.0.1:58442, datanodeUuid=21a56a47-4ffc-4762-a538-df52f644cdc7, infoPort=49125, infoSecurePort=0, ipcPort=53774, storageInfo=lv=-56;cid=testClusterID;nsid=818447236;c=0), blocks: 0, hasStaleStorage: true, processing time: 70 msecs
   [junit4]   2> 715361 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1a2f461db5c28e: from storage DS-bab6ae00-2ba5-4efb-9333-2d403c9fc9d9 node DatanodeRegistration(127.0.0.1:59042, datanodeUuid=e6596a48-1289-40aa-90bb-54cd35511ff3, infoPort=52725, infoSecurePort=0, ipcPort=53689, storageInfo=lv=-56;cid=testClusterID;nsid=818447236;c=0), blocks: 0, hasStaleStorage: true, processing time: 18 msecs
   [junit4]   2> 715361 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1a2f461e809dc3: from storage DS-60be838a-2b21-47f0-b0e6-c158f396eea8 node DatanodeRegistration(127.0.0.1:58442, datanodeUuid=21a56a47-4ffc-4762-a538-df52f644cdc7, infoPort=49125, infoSecurePort=0, ipcPort=53774, storageInfo=lv=-56;cid=testClusterID;nsid=818447236;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 715380 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1a2f461db5c28e: from storage DS-b550d8f6-9c95-4aa3-ab21-6b0bcf67387b node DatanodeRegistration(127.0.0.1:59042, datanodeUuid=e6596a48-1289-40aa-90bb-54cd35511ff3, infoPort=52725, infoSecurePort=0, ipcPort=53689, storageInfo=lv=-56;cid=testClusterID;nsid=818447236;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 719515 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 719516 INFO  (Thread-412) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 719516 INFO  (Thread-412) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 719716 ERROR (Thread-412) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 719743 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.ZkTestServer start zk server on port:51856
   [junit4]   2> 719813 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ee741d7520000, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 720042 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 720046 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 720061 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 720069 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 720069 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 720081 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 720089 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 720119 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 720125 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 720133 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 720141 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 720165 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 721674 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1aca5b1{/,null,AVAILABLE}
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5ab6f743{HTTP/1.1,[http/1.1]}{127.0.0.1:35965}
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.e.j.s.Server Started @749725ms
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, hostPort=35965, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/control-001/cores}
   [junit4]   2> 721682 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 721682 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 721683 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 721683 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-04T12:00:07.426Z
   [junit4]   2> 721897 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 721897 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/control-001/solr.xml
   [junit4]   2> 721919 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@456c89fd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 722929 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51856/solr
   [junit4]   2> 724254 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 724254 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35965_
   [junit4]   2> 724257 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.c.Overseer Overseer (id=98770511820095492-127.0.0.1:35965_-n_0000000000) starting
   [junit4]   2> 724301 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35965_
   [junit4]   2> 725440 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@456c89fd
   [junit4]   2> 725592 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@456c89fd
   [junit4]   2> 725592 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@456c89fd
   [junit4]   2> 725621 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [n:127.0.0.1:35965_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/control-001/cores
   [junit4]   2> 725653 INFO  (zkCallback-238-thread-1-processing-n:127.0.0.1:35965_) [n:127.0.0.1:35965_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 725861 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 725862 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[93F883ACB19AA413]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:51856/solr ready
   [junit4]   2> 725923 INFO  (qtp576116407-1734) [n:127.0.0.1:35965_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35965_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 725969 INFO  (OverseerThreadFactory-621-thread-1-processing-n:127.0.0.1:35965_) [n:127.0.0.1:35965_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 726265 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 726266 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 726421 INFO  (zkCallback-238-thread-1-processing-n:127.0.0.1:35965_) [n:127.0.0.1:35965_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 727430 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 727779 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 728934 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 729646 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 729651 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@456c89fd
   [junit4]   2> 729778 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:37663/solr_hdfs_home
   [junit4]   2> 729778 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 729778 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 729778 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_93F883ACB19AA413-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:37663/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 729780 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:37663/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 730011 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 730011 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 730011 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
   [junit4]   2> 739237 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 739521 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:37663/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 740593 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:37663/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 740995 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 740996 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 741049 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 741049 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3ba56b3a
   [junit4]   2> 744561 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:59042 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bab6ae00-2ba5-4efb-9333-2d403c9fc9d9:NORMAL:127.0.0.1:59042|RBW], ReplicaUC[[DISK]DS-6a655397-f967-4ddb-8113-40d611b6dddb:NORMAL:127.0.0.1:58442|RBW]]} size 72
   [junit4]   2> 744562 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58442 is added to blk_1073741825_1001 size 72
   [junit4]   2> 745063 WARN  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 745521 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 745521 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 745521 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 747033 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 747033 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 747072 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@fb53b6e
   [junit4]   2> 749846 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@24fa1a67[control_collection_shard1_replica_n1] main]
   [junit4]   2> 749855 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 749855 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 749858 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 749860 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580328220730851328
   [junit4]   2> 749936 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 749936 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 749936 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35965/control_collection_shard1_replica_n1/
   [junit4]   2> 749936 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 749936 INFO  (qtp576116407-1732) [n:127.0.0.1:35965_ c:control_collection s:shard1  x:control_collection_shard1_re

[...truncated too long message...]

DistributedZkTest]
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at <a href="http://127.0.0.1:33552:">http://127.0.0.1:33552: Cannot unload non-existent core [multiunload12]
   [junit4]    > at __randomizedtesting.SeedInfo.seed([93F883ACB19AA413]:0)
   [junit4]    > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]    > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]    > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]    > at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    > at org.apache.solr.cloud.UnloadDistributedZkTest.lambda$testUnloadLotsOfCores$0(UnloadDistributedZkTest.java:362)
   [junit4]    > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 6578904 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 6579641 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 6579773 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:58126) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 6579773 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:58126) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-2105895430-10.41.0.5-1507123971539 (Datanode Uuid 211baa6c-b20b-453c-9b62-0d917ca38c71) service to localhost/127.0.0.1:58126
   [junit4]   2> 6579774 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 6580041 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 6580086 WARN  (2064534012@qtp-608204702-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46878) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 6580086 WARN  (2064534012@qtp-608204702-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46878) [    ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 6580127 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:58126) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 6580127 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_93F883ACB19AA413-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:58126) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-2105895430-10.41.0.5-1507123971539 (Datanode Uuid 116c9784-0fba-48ba-9553-3635053949d3) service to localhost/127.0.0.1:58126
   [junit4]   2> 6580240 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 6581736 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[93F883ACB19AA413]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Oct 04, 2017 1:37:45 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=PostingsFormat(name=Memory), a_t=FST50, id=PostingsFormat(name=Memory), text=BlockTreeOrds(blocksize=128)}, docValues:{other_tl1=DocValuesFormat(name=Asserting), range_facet_l_dv=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Asserting), intDefault=DocValuesFormat(name=Asserting), id_i1=DocValuesFormat(name=Lucene70), a_i1=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=222, maxMBSortInHeap=7.413352655513397, sim=RandomSimilarity(queryNorm=true): {}, locale=id, timezone=Canada/Saskatchewan
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=343507552,total=528482304
   [junit4]   2> NOTE: All tests run in this JVM: [TestCharFilters, TestStressUserVersions, DocValuesMissingTest, ClusterStateUpdateTest, BlockJoinFacetSimpleTest, TestGraphTermsQParserPlugin, TestManagedSchemaAPI, PropertiesRequestHandlerTest, AliasIntegrationTest, TestManagedSynonymGraphFilterFactory, RemoteQueryErrorTest, TestRandomCollapseQParserPlugin, TestPartialUpdateDeduplication, TestSegmentSorting, TestRawResponseWriter, TestCursorMarkWithoutUniqueKey, HdfsDirectoryTest, TestFileDictionaryLookup, TestPullReplica, ShardRoutingCustomTest, SchemaVersionSpecificBehaviorTest, SolrSlf4jReporterTest, CdcrUpdateLogTest, CollectionReloadTest, SolrMetricReporterTest, ConvertedLegacyTest, HdfsSyncSliceTest, RestartWhileUpdatingTest, DisMaxRequestHandlerTest, SpellCheckCollatorWithCollapseTest, HdfsRecoverLeaseTest, DocValuesNotIndexedTest, ZkControllerTest, TestHashPartitioner, TestFuzzyAnalyzedSuggestions, TestPerFieldSimilarityWithDefaultOverride, HttpSolrCallGetCoreTest, ScriptEngineTest, TestSolrQueryResponse, DocExpirationUpdateProcessorFactoryTest, TestDistribDocBasedVersion, TestLRUStatsCache, DistributedExpandComponentTest, IndexSchemaTest, MoveReplicaTest, TestNumericTokenStream, TestBM25SimilarityFactory, TestTrackingShardHandlerFactory, TestRequestForwarding, TestSimpleQParserPlugin, TestHdfsUpdateLog, CoreAdminHandlerTest, TermVectorComponentTest, TestConfigSetProperties, PreAnalyzedUpdateProcessorTest, TestReplicationHandlerBackup, TestGraphMLResponseWriter, TestExpandComponent, BasicFunctionalityTest, TestCloudPivotFacet, HdfsUnloadDistributedZkTest]
   [junit4] Completed [600/733 (6!)] on J1 in 298.30s, 1 test, 1 error <<< FAILURES!

[...truncated 47035 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec]
     [exec] Crawl/parse...
     [exec]
     [exec] Verify...
     [exec]
     [exec] file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/docs/quickstart.html
     [exec]   BAD EXTERNAL LINK: https://lucene.apache.org/solr/guide/solr-tutorial.html
     [exec]
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:834: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:101: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:669: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:682: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:2570: exec returned: 1

Total time: 908 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 57 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/57/

9 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([75F854A4F550C671]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([75F854A4F550C671]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:99D3EF8B38E29421]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:     1) Thread[id=132397, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    2) Thread[id=132668, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    3) Thread[id=132671, name=zkCallback-25888-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=132396, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    5) Thread[id=132734, name=zkCallback-25888-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=132398, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)    7) Thread[id=132733, name=zkCallback-25888-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=132667, name=StoppableIndexingThread-SendThread(127.0.0.1:40618), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
   1) Thread[id=132397, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   2) Thread[id=132668, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   3) Thread[id=132671, name=zkCallback-25888-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=132396, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   5) Thread[id=132734, name=zkCallback-25888-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=132398, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
   7) Thread[id=132733, name=zkCallback-25888-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=132667, name=StoppableIndexingThread-SendThread(127.0.0.1:40618), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=132397, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Throwable.fillInStackTrace(Native Method)         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)         at java.lang.Throwable.<init>(Throwable.java:265)         at java.lang.Exception.<init>(Exception.java:66)         at java.lang.RuntimeException.<init>(RuntimeException.java:62)         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)         at org.apache.http.util.Asserts.check(Asserts.java:34)         at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:189)         at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:257)         at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:176)         at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)         at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)         at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)         at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)         at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)         at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    2) Thread[id=132671, name=zkCallback-25888-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=132396, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)         at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)         at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)         at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)         at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)         at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)         at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    4) Thread[id=132734, name=zkCallback-25888-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=132733, name=zkCallback-25888-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=132667, name=StoppableIndexingThread-SendThread(127.0.0.1:40618), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=132397, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Throwable.fillInStackTrace(Native Method)
        at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
        at java.lang.Throwable.<init>(Throwable.java:265)
        at java.lang.Exception.<init>(Exception.java:66)
        at java.lang.RuntimeException.<init>(RuntimeException.java:62)
        at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
        at org.apache.http.util.Asserts.check(Asserts.java:34)
        at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:189)
        at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:257)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:176)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   2) Thread[id=132671, name=zkCallback-25888-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=132396, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
        at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
        at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
        at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
        at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   4) Thread[id=132734, name=zkCallback-25888-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=132733, name=zkCallback-25888-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=132667, name=StoppableIndexingThread-SendThread(127.0.0.1:40618), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9]:0)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1507347347538,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1507347347538,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1507347347538,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, _9.cfe, _9.cfs, _9.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1507347347538,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1507347347538,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1507347347538,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, _9.cfe, _9.cfs, _9.si, segments_3]}]>
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:3450CB61E656F7DA]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
KeeperErrorCode = Session expired for /overseer_elect/leader

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:99D3EF8B38E29421]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderId(OverseerTaskProcessor.java:374)
        at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderNode(OverseerTaskProcessor.java:365)
        at org.apache.solr.cloud.RollingRestartTest.waitUntilOverseerDesignateIsLeader(RollingRestartTest.java:137)
        at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
        at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:D86ADDCE3BB874A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.TestTlogReplica.assertCopyOverOldUpdates(TestTlogReplica.java:909)
        at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:501)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:99D3EF8B38E29421]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10848 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> ott 07, 2017 10:18:53 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    2) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    3) Thread[id=13, name=SUITE-TestGeo3DPoint-seed#[75F854A4F550C671], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    4) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[75F854A4F550C671], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:480)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=75F854A4F550C671 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=it-IT -Dtests.timezone=Australia/Eucla -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   7106s J0 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([75F854A4F550C671]:0)
   [junit4]   2> ott 07, 2017 10:18:53 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> ott 07, 2017 10:19:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[75F854A4F550C671], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersectionBounds(Plane.java:1073)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1214)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addIntersection(XYZBounds.java:262)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:486)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ott 07, 2017 10:19:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[75F854A4F550C671], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/spatial3d/test/J0/temp/lucene.spatial3d.TestGeo3DPoint_75F854A4F550C671-001
   [junit4]   2> Oct 07, 2017 1:34:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=FSTOrd50}, docValues:{id=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1454, maxMBSortInHeap=6.510090742005915, sim=RandomSimilarity(queryNorm=false): {}, locale=it-IT, timezone=Australia/Eucla
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=200143840,total=461373440
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DDocValues, TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=75F854A4F550C671 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=it-IT -Dtests.timezone=Australia/Eucla -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([75F854A4F550C671]:0)
   [junit4] Completed [15/15 (1!)] on J0 in 7220.57s, 6 tests, 2 errors <<< FAILURES!

[...truncated 2814 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTlogReplica
   [junit4]   2> 4320999 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/init-core-data-001
   [junit4]   2> 4320999 WARN  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 4321000 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4321013 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 4321014 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001
   [junit4]   2> 4321014 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4321026 INFO  (Thread-7608) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4321026 INFO  (Thread-7608) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4321106 ERROR (Thread-7608) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4321134 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:37503
   [junit4]   2> 4321322 INFO  (jetty-launcher-5380-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 4321322 INFO  (jetty-launcher-5380-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 4321323 INFO  (jetty-launcher-5380-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7794dcf4{/solr,null,AVAILABLE}
   [junit4]   2> 4321323 INFO  (jetty-launcher-5380-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@77d40620{SSL,[ssl, http/1.1]}{127.0.0.1:33132}
   [junit4]   2> 4321323 INFO  (jetty-launcher-5380-thread-1) [    ] o.e.j.s.Server Started @4334971ms
   [junit4]   2> 4321323 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33132}
   [junit4]   2> 4321324 ERROR (jetty-launcher-5380-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4321324 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 4321324 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4321324 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4321324 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-07T02:55:02.472Z
   [junit4]   2> 4321324 INFO  (jetty-launcher-5380-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@762c1deb{/solr,null,AVAILABLE}
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@79627a7c{SSL,[ssl, http/1.1]}{127.0.0.1:33036}
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.e.j.s.Server Started @4334973ms
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33036}
   [junit4]   2> 4321325 ERROR (jetty-launcher-5380-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4321325 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-07T02:55:02.473Z
   [junit4]   2> 4321498 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 4321509 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 4321573 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ef4c1e81f0002, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 4321576 INFO  (jetty-launcher-5380-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37503/solr
   [junit4]   2> 4321577 INFO  (jetty-launcher-5380-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37503/solr
   [junit4]   2> 4321608 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ef4c1e81f0004, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 4321750 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4321769 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33036_solr
   [junit4]   2> 4321770 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.c.Overseer Overseer (id=98785355508940805-127.0.0.1:33036_solr-n_0000000000) starting
   [junit4]   2> 4321830 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33036_solr
   [junit4]   2> 4321830 INFO  (zkCallback-5391-thread-1-processing-n:127.0.0.1:33036_solr) [n:127.0.0.1:33036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4321973 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4321974 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4321994 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33132_solr
   [junit4]   2> 4321995 INFO  (zkCallback-5391-thread-1-processing-n:127.0.0.1:33036_solr) [n:127.0.0.1:33036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4322028 INFO  (zkCallback-5392-thread-1-processing-n:127.0.0.1:33132_solr) [n:127.0.0.1:33132_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4322137 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33036.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322161 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33036.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322161 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33036.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322174 INFO  (jetty-launcher-5380-thread-2) [n:127.0.0.1:33036_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node2/.
   [junit4]   2> 4322411 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33132.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322442 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33132.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322442 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33132.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4322443 INFO  (jetty-launcher-5380-thread-1) [n:127.0.0.1:33132_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node1/.
   [junit4]   2> 4322521 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4322522 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37503/solr ready
   [junit4]   2> 4322523 INFO  (SUITE-TestTlogReplica-seed#[1187D051961EF9D9]-worker) [    ] o.a.s.c.TestTlogReplica Using legacyCloud?: true
   [junit4]   2> 4322675 INFO  (qtp1101234141-21761) [n:127.0.0.1:33036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4322676 INFO  (qtp1101234141-21761) [n:127.0.0.1:33036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 4322694 INFO  (TEST-TestTlogReplica.testRemoveLeader-seed#[1187D051961EF9D9]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRemoveLeader
   [junit4]   2> 4322717 INFO  (qtp1101234141-21757) [n:127.0.0.1:33036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_remove_leader&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4322752 INFO  (OverseerThreadFactory-5017-thread-1-processing-n:127.0.0.1:33036_solr) [n:127.0.0.1:33036_solr    ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_remove_leader
   [junit4]   2> 4323008 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 4323008 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 4323168 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 4323168 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 4323365 INFO  (zkCallback-5391-thread-2-processing-n:127.0.0.1:33036_solr) [n:127.0.0.1:33036_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4323365 INFO  (zkCallback-5392-thread-1-processing-n:127.0.0.1:33132_solr) [n:127.0.0.1:33132_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4324241 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 4324284 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_remove_leader_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 4324286 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 4324286 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_remove_leader_shard1_replica_t1' using configuration from collection tlog_replica_test_remove_leader, trusted=true
   [junit4]   2> 4324286 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33036.solr.core.tlog_replica_test_remove_leader.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_remove_leader.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4324287 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4324287 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_remove_leader_shard1_replica_t1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node2/tlog_replica_test_remove_leader_shard1_replica_t1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node2/./tlog_replica_test_remove_leader_shard1_replica_t1/data/]
   [junit4]   2> 4324345 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 4324410 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_remove_leader_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 4324412 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 4324412 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_remove_leader_shard1_replica_t2' using configuration from collection tlog_replica_test_remove_leader, trusted=true
   [junit4]   2> 4324425 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33132.solr.core.tlog_replica_test_remove_leader.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_remove_leader.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac94aa0
   [junit4]   2> 4324425 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4324426 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_remove_leader_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node1/tlog_replica_test_remove_leader_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_1187D051961EF9D9-001/tempDir-001/node1/./tlog_replica_test_remove_leader_shard1_replica_t2/data/]
   [junit4]   2> 4324499 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4324499 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4324592 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4324592 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4324594 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@59d9314b[tlog_replica_test_remove_leader_shard1_replica_t1] main]
   [junit4]   2> 4324594 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 4324612 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 4324612 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4324612 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580565691902197760
   [junit4]   2> 4324647 INFO  (searcherExecutor-5022-thread-1-processing-n:127.0.0.1:33036_solr x:tlog_replica_test_remove_leader_shard1_replica_t1 s:shard1 c:tlog_replica_test_remove_leader) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t1] Registered new searcher Searcher@59d9314b[tlog_replica_test_remove_leader_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4324662 INFO  (qtp1101234141-21751) [n:127.0.0.1:33036_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 4324722 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4324722 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4324723 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4324723 INFO  (qtp1670006388-21762) [n:127.0.0.1:33132_solr c:tlog_replica_test_remove_leader s:shard1  x:tlog_replica_test_remove_leader_shard1_replica_t2] o.a.s.u.CommitTracker Soft

[...truncated too long message...]

        13 /solr/collections
   [junit4]   2> 8 /solr/live_nodes
   [junit4]   2> 5 /solr/overseer/queue
   [junit4]   2> 5 /solr/overseer/collection-queue-work
   [junit4]   2> 3 /solr/overseer/queue-work
   [junit4]   2>
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SharedFSAutoReplicaFailoverTest -Dtests.method=test -Dtests.seed=1187D051961EF9D9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-MX -Dtests.timezone=Europe/Belfast -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  117s J2 | SharedFSAutoReplicaFailoverTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<2> but was:<3>
   [junit4]    > at __randomizedtesting.SeedInfo.seed([1187D051961EF9D9:99D3EF8B38E29421]:0)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
   [junit4]    > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 8097509 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1187D051961EF9D9]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 8097568 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1187D051961EF9D9]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 8097669 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32902) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 8097669 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32902) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-957641662-127.0.0.1-1507348554410 (Datanode Uuid cb2b8202-cfb8-4e9e-89ee-706fac124a74) service to lucene2-us-west.apache.org/127.0.0.1:32902
   [junit4]   2> 8097670 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1187D051961EF9D9]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 8097718 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1187D051961EF9D9]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 8097819 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32902) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 8097820 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32902) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-957641662-127.0.0.1-1507348554410 (Datanode Uuid 2c39b754-8535-450e-87a5-b00391a327f0) service to lucene2-us-west.apache.org/127.0.0.1:32902
   [junit4]   2> 8097853 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1187D051961EF9D9]-worker) [    ] o.m.log Stopped HttpServer2$[hidden email]:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1187D051961EF9D9-001
   [junit4]   2> Oct 07, 2017 3:57:59 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=es-MX, timezone=Europe/Belfast
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=192798200,total=523239424
   [junit4]   2> NOTE: All tests run in this JVM: [TestNumericTokenStream, TestIndexSearcher, TestLegacyNumericRangeQueryBuilder, TestSolrCoreProperties, TestShortCircuitedRequests, TestConfigSetsAPI, PolyFieldTest, TestSolrCoreParser, RankQueryTest, TestRebalanceLeaders, AutoCommitTest, CollectionsAPISolrJTest, SuggesterFSTTest, TestDownShardTolerantSearch, JSONWriterTest, TestMinMaxOnMultiValuedField, BlobRepositoryCloudTest, ReplaceNodeTest, TestTrie, TestConfigsApi, PropertiesRequestHandlerTest, DistributedFacetPivotWhiteBoxTest, TestCustomDocTransformer, UtilsToolTest, BadIndexSchemaTest, OverriddenZkACLAndCredentialsProvidersTest, DateMathParserTest, DeleteInactiveReplicaTest, TestReplicationHandlerBackup, SpellPossibilityIteratorTest, TestNamedUpdateProcessors, PeerSyncReplicationTest, TestFieldCollectionResource, TestFieldTypeResource, DistributedExpandComponentTest, TestXmlQParserPlugin, TestCustomStream, DocValuesMissingTest, TestRecoveryHdfs, XsltUpdateRequestHandlerTest, TestLegacyFieldCache, CursorMarkTest, TestTrackingShardHandlerFactory, DeleteReplicaTest, TestSolrQueryResponse, StatsReloadRaceTest, BaseCdcrDistributedZkTest, TestSchemaManager, ResponseLogComponentTest, TestEmbeddedSolrServerConstructors, TestPullReplicaErrorHandling, CopyFieldTest, CdcrUpdateLogTest, ConfigureRecoveryStrategyTest, HttpSolrCallGetCoreTest, PrimitiveFieldTypeTest, PingRequestHandlerTest, SparseHLLTest, V2StandaloneTest, HdfsBasicDistributedZk2Test, TestReversedWildcardFilterFactory, TestFastWriter, TestExpandComponent, HdfsDirectoryTest, TestSuggestSpellingConverter, FullHLLTest, TestJsonRequest, DistanceUnitsTest, LeaderFailoverAfterPartitionTest, DirectUpdateHandlerTest, TestStreamBody, TestManagedSchemaAPI, UpdateLogTest, TestStressUserVersions, TestGroupingSearch, ShardSplitTest, TestLeaderElectionWithEmptyReplica, ConnectionReuseTest, TestSolrConfigHandlerConcurrent, SearchHandlerTest, DocExpirationUpdateProcessorFactoryTest, SuggesterTest, TestUniqueKeyFieldResource, TestSimpleQParserPlugin, SpellCheckCollatorWithCollapseTest, TestHdfsUpdateLog, ForceLeaderTest, ReturnFieldsTest, CoreMergeIndexesAdminHandlerTest, OverseerRolesTest, TestClusterProperties, HdfsRecoverLeaseTest, TestBulkSchemaConcurrent, CleanupOldIndexTest, ChangedSchemaMergeTest, BlockCacheTest, UnloadDistributedZkTest, SubstringBytesRefFilterTest, ZkFailoverTest, OverseerTest, FullSolrCloudDistribCmdsTest, LeaderInitiatedRecoveryOnShardRestartTest, PKIAuthenticationIntegrationTest, TestSolrCloudWithDelegationTokens, TestRuleBasedAuthorizationPlugin, MultiTermTest, SolrMetricReporterTest, TestTlogReplica, TestUnifiedSolrHighlighter, ReplicaListTransformerTest, TestLMJelinekMercerSimilarityFactory, TestRawResponseWriter, TestUseDocValuesAsStored2, DistribCursorPagingTest, ConnectionManagerTest, RollingRestartTest, PreAnalyzedUpdateProcessorTest, ClassificationUpdateProcessorIntegrationTest, DirectUpdateHandlerOptimizeTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, TestRangeQuery, SolrRequestParserTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, SortByFunctionTest, DocValuesMultiTest, TestSolrDeletionPolicy1, TestSurroundQueryParser, DisMaxRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestStressRecovery, CdcrReplicationHandlerTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ConcurrentDeleteAndCreateCollectionTest, SharedFSAutoReplicaFailoverTest]
   [junit4] Completed [734/734 (5!)] on J2 in 134.55s, 1 test, 1 failure <<< FAILURES!

[...truncated 46862 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 58 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/58/

7 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
        at __randomizedtesting.SeedInfo.seed([485BB662982E7692:B11625CDA45B3B18]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([B0F38546A4706526]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([B0F38546A4706526]:0)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1507510437103,generation=2,filelist=[_x3.cfe, _x3.cfs, _x3.si, _x4.fdt, _x4.fdx, _x4.fnm, _x4.nvd, _x4.nvm, _x4.si, _x4_Lucene50_0.doc, _x4_Lucene50_0.tim, _x4_Lucene50_0.tip, _x8.cfe, _x8.cfs, _x8.si, _xa.cfe, _xa.cfs, _xa.si, _xb.cfe, _xb.cfs, _xb.si, _xc.fdt, _xc.fdx, _xc.fnm, _xc.nvd, _xc.nvm, _xc.si, _xc_Lucene50_0.doc, _xc_Lucene50_0.tim, _xc_Lucene50_0.tip, _xd.cfe, _xd.cfs, _xd.si, _xe.cfe, _xe.cfs, _xe.si, _xf.cfe, _xf.cfs, _xf.si, _xg.cfe, _xg.cfs, _xg.si, _xi.cfe, _xi.cfs, _xi.si, segments_2]}]> but was:<[{indexVersion=1507510437103,generation=2,filelist=[_x3.cfe, _x3.cfs, _x3.si, _x4.fdt, _x4.fdx, _x4.fnm, _x4.nvd, _x4.nvm, _x4.si, _x4_Lucene50_0.doc, _x4_Lucene50_0.tim, _x4_Lucene50_0.tip, _x8.cfe, _x8.cfs, _x8.si, _xa.cfe, _xa.cfs, _xa.si, _xb.cfe, _xb.cfs, _xb.si, _xc.fdt, _xc.fdx, _xc.fnm, _xc.nvd, _xc.nvm, _xc.si, _xc_Lucene50_0.doc, _xc_Lucene50_0.tim, _xc_Lucene50_0.tip, _xd.cfe, _xd.cfs, _xd.si, _xe.cfe, _xe.cfs, _xe.si, _xf.cfe, _xf.cfs, _xf.si, _xg.cfe, _xg.cfs, _xg.si, _xi.cfe, _xi.cfs, _xi.si, segments_2]}, {indexVersion=1507510437103,generation=3,filelist=[_xd.cfe, _xd.cfs, _xd.si, _xg.cfe, _xg.cfs, _xg.si, _xh.cfe, _xh.cfs, _xh.si, _xi.cfe, _xi.cfs, _xi.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1507510437103,generation=2,filelist=[_x3.cfe, _x3.cfs, _x3.si, _x4.fdt, _x4.fdx, _x4.fnm, _x4.nvd, _x4.nvm, _x4.si, _x4_Lucene50_0.doc, _x4_Lucene50_0.tim, _x4_Lucene50_0.tip, _x8.cfe, _x8.cfs, _x8.si, _xa.cfe, _xa.cfs, _xa.si, _xb.cfe, _xb.cfs, _xb.si, _xc.fdt, _xc.fdx, _xc.fnm, _xc.nvd, _xc.nvm, _xc.si, _xc_Lucene50_0.doc, _xc_Lucene50_0.tim, _xc_Lucene50_0.tip, _xd.cfe, _xd.cfs, _xd.si, _xe.cfe, _xe.cfs, _xe.si, _xf.cfe, _xf.cfs, _xf.si, _xg.cfe, _xg.cfs, _xg.si, _xi.cfe, _xi.cfs, _xi.si, segments_2]}]> but was:<[{indexVersion=1507510437103,generation=2,filelist=[_x3.cfe, _x3.cfs, _x3.si, _x4.fdt, _x4.fdx, _x4.fnm, _x4.nvd, _x4.nvm, _x4.si, _x4_Lucene50_0.doc, _x4_Lucene50_0.tim, _x4_Lucene50_0.tip, _x8.cfe, _x8.cfs, _x8.si, _xa.cfe, _xa.cfs, _xa.si, _xb.cfe, _xb.cfs, _xb.si, _xc.fdt, _xc.fdx, _xc.fnm, _xc.nvd, _xc.nvm, _xc.si, _xc_Lucene50_0.doc, _xc_Lucene50_0.tim, _xc_Lucene50_0.tip, _xd.cfe, _xd.cfs, _xd.si, _xe.cfe, _xe.cfs, _xe.si, _xf.cfe, _xf.cfs, _xf.si, _xg.cfe, _xg.cfs, _xg.si, _xi.cfe, _xi.cfs, _xi.si, segments_2]}, {indexVersion=1507510437103,generation=3,filelist=[_xd.cfe, _xd.cfs, _xd.si, _xg.cfe, _xg.cfs, _xg.si, _xh.cfe, _xh.cfs, _xh.si, _xi.cfe, _xi.cfs, _xi.si, segments_3]}]>
        at __randomizedtesting.SeedInfo.seed([485BB662982E7692:6D8CAD52E8667891]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestAuthenticationFramework.testBasics

Error Message:
Error from server at https://127.0.0.1:42617/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:42617/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([485BB662982E7692:7583184EA0C028E2]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
        at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42617/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:
expected:<2> but was:<5>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<5>
        at __randomizedtesting.SeedInfo.seed([485BB662982E7692:545ACBEFED8B0801]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.TestTlogReplica.assertCopyOverOldUpdates(TestTlogReplica.java:909)
        at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:501)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at __randomizedtesting.SeedInfo.seed([485BB662982E7692:C00F89B836D21B6A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
        at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10991 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> жовт. 09, 2017 9:21:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[B0F38546A4706526], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[B0F38546A4706526], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:800)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.StandardXYZSolid.<init>(StandardXYZSolid.java:176)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZSolidFactory.makeXYZSolid(XYZSolidFactory.java:64)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:847)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=B0F38546A4706526 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=uk-UA -Dtests.timezone=Australia/Adelaide -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7108s J1 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B0F38546A4706526]:0)
   [junit4]   2> жовт. 09, 2017 9:21:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> жовт. 09, 2017 9:22:06 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[B0F38546A4706526], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.StandardXYZSolid.<init>(StandardXYZSolid.java:179)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZSolidFactory.makeXYZSolid(XYZSolidFactory.java:64)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:847)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> жовт. 09, 2017 9:22:06 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[B0F38546A4706526], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/spatial3d/test/J1/temp/lucene.spatial3d.TestGeo3DPoint_B0F38546A4706526-001
   [junit4]   2> Oct 08, 2017 10:52:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Memory)}, docValues:{id=DocValuesFormat(name=Asserting), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=687, maxMBSortInHeap=6.4098379953638815, sim=RandomSimilarity(queryNorm=true): {}, locale=uk-UA, timezone=Australia/Adelaide
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=70659808,total=494927872
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=B0F38546A4706526 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=uk-UA -Dtests.timezone=Australia/Adelaide -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B0F38546A4706526]:0)
   [junit4] Completed [14/15 (1!)] on J1 in 7222.34s, 8 tests, 2 errors <<< FAILURES!

[...truncated 1964 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTlogReplica
   [junit4]   2> 405212 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/init-core-data-001
   [junit4]   2> 405213 WARN  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 405213 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 405214 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 405216 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001
   [junit4]   2> 405216 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 405232 INFO  (Thread-357) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 405232 INFO  (Thread-357) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 405318 ERROR (Thread-357) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 405342 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:46634
   [junit4]   2> 405762 INFO  (jetty-launcher-170-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 405762 INFO  (jetty-launcher-170-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 405838 INFO  (jetty-launcher-170-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@544deb66{/solr,null,AVAILABLE}
   [junit4]   2> 405945 INFO  (jetty-launcher-170-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@130efb6c{/solr,null,AVAILABLE}
   [junit4]   2> 406402 INFO  (jetty-launcher-170-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5ede6f1b{SSL,[ssl, http/1.1]}{127.0.0.1:39936}
   [junit4]   2> 406402 INFO  (jetty-launcher-170-thread-1) [    ] o.e.j.s.Server Started @418810ms
   [junit4]   2> 406402 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39936}
   [junit4]   2> 406403 ERROR (jetty-launcher-170-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 406403 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 406403 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 406403 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 406403 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-08T23:08:11.674Z
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2eb8eb7d{SSL,[ssl, http/1.1]}{127.0.0.1:45165}
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.e.j.s.Server Started @418918ms
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45165}
   [junit4]   2> 406511 ERROR (jetty-launcher-170-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 406511 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-08T23:08:11.782Z
   [junit4]   2> 406620 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 406641 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 406803 INFO  (jetty-launcher-170-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46634/solr
   [junit4]   2> 406925 INFO  (jetty-launcher-170-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46634/solr
   [junit4]   2> 407640 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 407693 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45165_solr
   [junit4]   2> 407694 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.c.Overseer Overseer (id=98795788078415878-127.0.0.1:45165_solr-n_0000000000) starting
   [junit4]   2> 407712 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 407751 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45165_solr
   [junit4]   2> 407752 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39936_solr
   [junit4]   2> 407753 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 407896 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 407896 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 409731 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39936.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 409805 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39936.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 409805 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39936.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 409806 INFO  (jetty-launcher-170-thread-1) [n:127.0.0.1:39936_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node1/.
   [junit4]   2> 410513 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45165.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 410569 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45165.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 410569 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45165.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 410570 INFO  (jetty-launcher-170-thread-2) [n:127.0.0.1:45165_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node2/.
   [junit4]   2> 411089 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 411090 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46634/solr ready
   [junit4]   2> 411105 INFO  (SUITE-TestTlogReplica-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
   [junit4]   2> 412458 INFO  (qtp1294704362-1297) [n:127.0.0.1:39936_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 412497 INFO  (qtp1294704362-1297) [n:127.0.0.1:39936_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=38
   [junit4]   2> 412569 INFO  (TEST-TestTlogReplica.testBasicLeaderElection-seed#[485BB662982E7692]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBasicLeaderElection
   [junit4]   2> 412571 INFO  (qtp1294704362-1296) [n:127.0.0.1:39936_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_basic_leader_election&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 412730 INFO  (OverseerThreadFactory-440-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_basic_leader_election
   [junit4]   2> 413295 INFO  (OverseerStateUpdate-98795788078415878-127.0.0.1:45165_solr-n_0000000000) [n:127.0.0.1:45165_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_basic_leader_election",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_basic_leader_election_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45165/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 413552 INFO  (OverseerStateUpdate-98795788078415878-127.0.0.1:45165_solr-n_0000000000) [n:127.0.0.1:45165_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_basic_leader_election",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_basic_leader_election_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39936/solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 414402 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_basic_leader_election_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_basic_leader_election&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 414403 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 414513 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_basic_leader_election_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_basic_leader_election&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 414513 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 414694 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 414702 INFO  (zkCallback-182-thread-2-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 414702 INFO  (zkCallback-180-thread-2-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 414710 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 415662 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 415808 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 415845 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_basic_leader_election_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 415919 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 415919 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_basic_leader_election_shard1_replica_t1' using configuration from collection tlog_replica_test_basic_leader_election, trusted=true
   [junit4]   2> 415975 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_basic_leader_election_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 415977 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 415977 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_basic_leader_election_shard1_replica_t2' using configuration from collection tlog_replica_test_basic_leader_election, trusted=true
   [junit4]   2> 415992 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45165.solr.core.tlog_replica_test_basic_leader_election.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_basic_leader_election.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 415992 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 415992 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_basic_leader_election_shard1_replica_t1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node2/tlog_replica_test_basic_leader_election_shard1_replica_t1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node2/./tlog_replica_test_basic_leader_election_shard1_replica_t1/data/]
   [junit4]   2> 416035 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39936.solr.core.tlog_replica_test_basic_leader_election.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_basic_leader_election.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dfe8c7c
   [junit4]   2> 416035 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 416035 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_basic_leader_election_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node1/tlog_replica_test_basic_leader_election_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_485BB662982E7692-001/tempDir-001/node1/./tlog_replica_test_basic_leader_election_shard1_replica_t2/data/]
   [junit4]   2> 416781 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 416781 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 416782 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 416782 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 416838 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@60bf70ab[tlog_replica_test_basic_leader_election_shard1_replica_t2] main]
   [junit4]   2> 416838 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 416839 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 416875 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 416875 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580732621411844096
   [junit4]   2> 416877 INFO  (searcherExecutor-446-thread-1-processing-n:127.0.0.1:39936_solr x:tlog_replica_test_basic_leader_election_shard1_replica_t2 s:shard1 c:tlog_replica_test_basic_leader_election r:core_node4) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_basic_leader_election_shard1_replica_t2] Registered new searcher Searcher@60bf70ab[tlog_replica_test_basic_leader_election_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 416911 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 416911 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 416912 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 416912 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 416929 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9998ms
   [junit4]   2> 416964 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@73e07108[tlog_replica_test_basic_leader_election_shard1_replica_t1] main]
   [junit4]   2> 417000 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 417002 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 417003 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 417037 INFO  (searcherExecutor-445-thread-1-processing-n:127.0.0.1:45165_solr x:tlog_replica_test_basic_leader_election_shard1_replica_t1 s:shard1 c:tlog_replica_test_basic_leader_election r:core_node3) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_basic_leader_election_shard1_replica_t1] Registered new searcher Searcher@73e07108[tlog_replica_test_basic_leader_election_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 417053 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580732621598490624
   [junit4]   2> 417074 INFO  (zkCallback-180-thread-2-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417075 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417075 INFO  (zkCallback-182-thread-2-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417075 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417435 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 417436 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 417436 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:39936/solr/tlog_replica_test_basic_leader_election_shard1_replica_t2/
   [junit4]   2> 417473 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_basic_leader_election_shard1_replica_t2 url=https://127.0.0.1:39936/solr START replicas=[https://127.0.0.1:45165/solr/tlog_replica_test_basic_leader_election_shard1_replica_t1/] nUpdates=100
   [junit4]   2> 417727 INFO  (qtp663392229-1288) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_basic_leader_election_shard1_replica_t1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=52
   [junit4]   2> 417784 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_basic_leader_election_shard1_replica_t2 url=https://127.0.0.1:39936/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 417784 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 417784 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 417784 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 417785 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_basic_leader_election_shard1_replica_t2 stopping background replication from leader
   [junit4]   2> 417787 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:39936/solr/tlog_replica_test_basic_leader_election_shard1_replica_t2/ shard1
   [junit4]   2> 417896 INFO  (zkCallback-180-thread-2-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417896 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417896 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417896 INFO  (zkCallback-182-thread-2-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 417947 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 417948 INFO  (qtp1294704362-1286) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_basic_leader_election_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_basic_leader_election&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3435
   [junit4]   2> 418078 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_basic_leader_election_shard1_replica_t1 starting background replication from leader
   [junit4]   2> 418078 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 418115 INFO  (zkCallback-180-thread-2-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418115 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418116 INFO  (zkCallback-182-thread-2-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418116 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418152 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 418152 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 418153 INFO  (qtp663392229-1292) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_basic_leader_election_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_basic_leader_election&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3751
   [junit4]   2> 418240 INFO  (qtp1294704362-1296) [n:127.0.0.1:39936_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 418355 INFO  (zkCallback-182-thread-2-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418355 INFO  (zkCallback-182-thread-1-processing-n:127.0.0.1:45165_solr) [n:127.0.0.1:45165_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418365 INFO  (zkCallback-180-thread-2-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418365 INFO  (zkCallback-180-thread-1-processing-n:127.0.0.1:39936_solr) [n:127.0.0.1:39936_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_basic_leader_election/state.json] for collection [tlog_replica_test_basic_leader_election] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 418814 INFO  (OverseerCollectionConfigSetProcessor-98795788078415878-127.0.0.1:45165_solr-n_0000000000) [n:127.0.0.1:45165_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 419117 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:39936/solr/tlog_replica_test_basic_leader_election_shard1_replica_t2/
   [junit4]   2> 419207 INFO  (qtp1294704362-1297) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_basic_leader_election_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=50
   [junit4]   2> 419208 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 419208 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 419208 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 419208 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 419208 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 419209 INFO  (indexFetcher-456-thread-1) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@113641a[tlog_replica_test_basic_leader_election_shard1_replica_t1] main]
   [junit4]   2> 419243 INFO  (qtp1294704362-1296) [n:127.0.0.1:39936_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_basic_leader_election&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=6671
   [junit4]   2> 419264 INFO  (searcherExecutor-445-thread-1-processing-n:127.0.0.1:45165_solr x:tlog_replica_test_basic_leader_election_shard1_replica_t1 s:shard1 c:tlog_replica_test_basic_leader_election r:core_node3) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_basic_leader_election_shard1_replica_t1] Registered new searcher Searcher@113641a[tlog_replica_test_basic_leader_election_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 419448 INFO  (qtp663392229-1287) [n:127.0.0.1:45165_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node3 x:tlog_replica_test_basic_leader_election_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_basic_leader_election_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1580732623993438208&distrib.from=https://127.0.0.1:39936/solr/tlog_replica_test_basic_leader_election_shard1_replica_t2/&wt=javabin&version=2}{deleteByQuery=*:* (-1580732623993438208)} 0 1
   [junit4]   2> 419860 INFO  (qtp1294704362-1284) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580732624541843456,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 419861 INFO  (qtp1294704362-1284) [n:127.0.0.1:39936_solr c:tlog_replica_test_basic_leader_election s:shard1 r:core_node4 x:tlog_replica_test_basic_leader_election_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 420021 INFO  (qtp663392229-1290) [n:127.0.0.1:45165_

[...truncated too long message...]

cutor-14314-thread-1) [n:127.0.0.1:42617_solr c:testcollection s:shard1 r:core_node4 x:testcollection_shard1_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=350025195
   [junit4]   2> 8245474 INFO  (coreCloseExecutor-14314-thread-2) [n:127.0.0.1:42617_solr c:testcollection s:shard2 r:core_node8 x:testcollection_shard2_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard2.replica_n6, tag=1158898361
   [junit4]   2> 8245474 INFO  (coreCloseExecutor-14314-thread-2) [n:127.0.0.1:42617_solr c:testcollection s:shard2 r:core_node8 x:testcollection_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4715d0f5: rootName = solr_42617, domain = solr.core.testcollection.shard2.replica_n6, service url = null, agent id = null] for registry solr.core.testcollection.shard2.replica_n6 / com.codahale.metrics.MetricRegistry@4377c06b
   [junit4]   2> 8245565 INFO  (jetty-closer-33294-thread-1) [    ] o.a.s.c.Overseer Overseer (id=98796300731875334-127.0.0.1:41543_solr-n_0000000000) closing
   [junit4]   2> 8245573 INFO  (OverseerStateUpdate-98796300731875334-127.0.0.1:41543_solr-n_0000000000) [n:127.0.0.1:41543_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:41543_solr
   [junit4]   2> 8245597 INFO  (coreCloseExecutor-14314-thread-2) [n:127.0.0.1:42617_solr c:testcollection s:shard2 r:core_node8 x:testcollection_shard2_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard2.leader, tag=1158898361
   [junit4]   2> 8247132 WARN  (zkCallback-33304-thread-1-processing-n:127.0.0.1:42617_solr) [n:127.0.0.1:42617_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 8247220 WARN  (zkCallback-33305-thread-3-processing-n:127.0.0.1:41543_solr) [n:127.0.0.1:41543_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 8247220 INFO  (jetty-closer-33294-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4dc86d03{/solr,null,UNAVAILABLE}
   [junit4]   2> 8247220 INFO  (jetty-closer-33294-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@541732a7{/solr,null,UNAVAILABLE}
   [junit4]   2> 8247221 ERROR (SUITE-TestAuthenticationFramework-seed#[485BB662982E7692]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 8247222 INFO  (SUITE-TestAuthenticationFramework-seed#[485BB662982E7692]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34690 34690
   [junit4]   2> 8247297 INFO  (Thread-13209) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34690 34690
   [junit4]   2> 8247298 WARN  (Thread-13209) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 4 /solr/configs/solrCloudCollectionConfig
   [junit4]   2> 3 /solr/aliases.json
   [junit4]   2> 2 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 16 /solr/collections/testcollection/state.json
   [junit4]   2> 3 /solr/clusterstate.json
   [junit4]   2> 3 /solr/clusterprops.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 3 /solr/live_nodes
   [junit4]   2> 3 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestAuthenticationFramework_485BB662982E7692-001
   [junit4]   2> Oct 09, 2017 1:18:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=PostingsFormat(name=Asserting), id=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{}, maxPointsInLeafNode=1969, maxMBSortInHeap=5.395317729382315, sim=RandomSimilarity(queryNorm=true): {}, locale=es-BO, timezone=Asia/Tbilisi
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=177559928,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [TestBulkSchemaAPI, TestImplicitCoreProperties, StressHdfsTest, TestConfigSetsAPIZkFailure, JvmMetricsTest, SpatialRPTFieldTypeTest, CloneFieldUpdateProcessorFactoryTest, TestFieldCacheSort, OpenCloseCoreStressTest, TestFastLRUCache, SolrShardReporterTest, TestSortByMinMaxFunction, BasicFunctionalityTest, SpellCheckComponentTest, TestStressRecovery, TestPullReplica, RestartWhileUpdatingTest, TestQueryTypes, TestManagedSynonymFilterFactory, AnalyticsQueryTest, TestEmbeddedSolrServerAdminHandler, TemplateUpdateProcessorTest, TestPushWriter, TestDelegationWithHadoopAuth, TestOverriddenPrefixQueryForCustomFieldType, SliceStateTest, TestCollectionsAPIViaSolrCloudCluster, TestComplexPhraseQParserPlugin, TestCoreDiscovery, TestPostingsSolrHighlighter, TestInitQParser, SolrMetricManagerTest, ZkSolrClientTest, AnalysisErrorHandlingTest, DistribJoinFromCollectionTest, DirectSolrConnectionTest, LeaderInitiatedRecoveryOnCommitTest, AnalysisAfterCoreReloadTest, TestFieldTypeCollectionResource, AliasIntegrationTest, TestPointFields, ChaosMonkeySafeLeaderTest, TestCorePropertiesReload, HighlighterMaxOffsetTest, DateRangeFieldTest, TestElisionMultitermQuery, TestDynamicFieldResource, TestSolrCloudSnapshots, ClusterStateUpdateTest, TestSegmentSorting, TestLeaderInitiatedRecoveryThread, BadComponentTest, ZkCLITest, TestBM25SimilarityFactory, RulesTest, DisMaxRequestHandlerTest, TestSearchPerf, TestPolicyCloud, TestNumericTerms64, DeleteStatusTest, ZkControllerTest, TestSolrIndexConfig, RemoteQueryErrorTest, HLLUtilTest, DistributedFacetPivotLargeTest, DistributedMLTComponentTest, TestSerializedLuceneMatchVersion, ManagedSchemaRoundRobinCloudTest, TestScoreJoinQPScore, TestTolerantUpdateProcessorRandomCloud, ExplicitHLLTest, SimpleMLTQParserTest, ZkStateWriterTest, TestHdfsBackupRestoreCore, TestInPlaceUpdatesStandalone, TestRetrieveFieldsOptimizer, TestDistributedStatsComponentCardinality, ResourceLoaderTest, CdcrVersionReplicationTest, TestMiniSolrCloudClusterSSL, DistributedTermsComponentTest, TestRandomRequestDistribution, TestNonDefinedSimilarityFactory, TestRequestForwarding, TestXIncludeConfig, TestLegacyTerms, SoftAutoCommitTest, SegmentsInfoRequestHandlerTest, CollectionReloadTest, TestValueSourceCache, TestBinaryResponseWriter, TestRTGBase, TestSchemaResource, TestCharFilters, ScriptEngineTest, TestFilteredDocIdSet, TestSearcherReuse, FileUtilsTest, TestSolrCloudWithSecureImpersonation, BasicAuthStandaloneTest, CoreAdminHandlerTest, TestXmlQParser, BufferStoreTest, TestSolr4Spatial2, SpatialFilterTest, BooleanFieldTest, TestBlendedInfixSuggestions, NoCacheHeaderTest, DistributedIntervalFacetingTest, ConfigSetsAPITest, CoreAdminRequestStatusTest, V2ApiIntegrationTest, TestStressLiveNodes, MoveReplicaTest, SystemInfoHandlerTest, SolrCmdDistributorTest, IndexSchemaRuntimeFieldTest, TestSimpleTrackingShardHandler, TestCloudDeleteByQuery, TestCursorMarkWithoutUniqueKey, CacheHeaderTest, CollectionsAPIDistributedZkTest, OverseerTest, TestReplicationHandler, TestSort, TestFunctionQuery, DirectUpdateHandlerTest, HighlighterTest, DistributedQueryElevationComponentTest, SolrRequestParserTest, SuggesterTSTTest, SuggesterTest, SuggesterWFSTTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, DebugComponentTest, IndexBasedSpellCheckerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, TestIndexingPerformance, MoreLikeThisHandlerTest, TestSolrQueryParser, LoggingHandlerTest, IndexSchemaTest, TestCollationField, TestCSVResponseWriter, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestLFUCache, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestLRUCache, TestUtils, TestDocumentBuilder, SolrTestCaseJ4Test, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AssignBackwardCompatibilityTest, AsyncCallRequestStatusResponseTest, CdcrReplicationDistributedZkTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CleanupOldIndexTest, CollectionStateFormat2Test, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, DeleteNodeTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MoveReplicaHDFSUlogDirTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, TestAuthenticationFramework]
   [junit4] Completed [461/734 (5!)] on J1 in 20.70s, 1 test, 1 error <<< FAILURES!

[...truncated 47881 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 59 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/59/

6 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.NoFacetCloudTest

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
        at __randomizedtesting.SeedInfo.seed([F3905631EEDD9F00]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
        at org.apache.solr.analytics.AbstractAnalyticsStatsCloudTest.setupCluster(AbstractAnalyticsStatsCloudTest.java:76)
        at org.apache.solr.analytics.NoFacetCloudTest.populate(NoFacetCloudTest.java:62)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest:     1) Thread[id=2962, name=zkCallback-420-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest:
   1) Thread[id=2962, name=zkCallback-420-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2962, name=zkCallback-420-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2962, name=zkCallback-420-thread-1, state=TIMED_WAITING, group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 seconds
        at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E:1BAE1DAC9CE2D186]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
        at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
        at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E:1BAE1DAC9CE2D186]:0)
        at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1507625043838,generation=2,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j17.cfe, _j17.cfs, _j17.si, _j18.fdt, _j18.fdx, _j18.fnm, _j18.nvd, _j18.nvm, _j18.si, _j18_FST50_0.doc, _j18_FST50_0.tfp, _j19.fdt, _j19.fdx, _j19.fnm, _j19.nvd, _j19.nvm, _j19.si, _j19_FST50_0.doc, _j19_FST50_0.tfp, _j1a.fdt, _j1a.fdx, _j1a.fnm, _j1a.nvd, _j1a.nvm, _j1a.si, _j1a_FST50_0.doc, _j1a_FST50_0.tfp, _j1b.fdt, _j1b.fdx, _j1b.fnm, _j1b.nvd, _j1b.nvm, _j1b.si, _j1b_FST50_0.doc, _j1b_FST50_0.tfp, _j1c.fdt, _j1c.fdx, _j1c.fnm, _j1c.nvd, _j1c.nvm, _j1c.si, _j1c_FST50_0.doc, _j1c_FST50_0.tfp, _j1d.fdt, _j1d.fdx, _j1d.fnm, _j1d.nvd, _j1d.nvm, _j1d.si, _j1d_FST50_0.doc, _j1d_FST50_0.tfp, _j1e.fdt, _j1e.fdx, _j1e.fnm, _j1e.nvd, _j1e.nvm, _j1e.si, _j1e_FST50_0.doc, _j1e_FST50_0.tfp, _j1f.fdt, _j1f.fdx, _j1f.fnm, _j1f.nvd, _j1f.nvm, _j1f.si, _j1f_FST50_0.doc, _j1f_FST50_0.tfp, _j1g.fdt, _j1g.fdx, _j1g.fnm, _j1g.nvd, _j1g.nvm, _j1g.si, _j1g_FST50_0.doc, _j1g_FST50_0.tfp, _j1h.fdt, _j1h.fdx, _j1h.fnm, _j1h.nvd, _j1h.nvm, _j1h.si, _j1h_FST50_0.doc, _j1h_FST50_0.tfp, _j1i.fdt, _j1i.fdx, _j1i.fnm, _j1i.nvd, _j1i.nvm, _j1i.si, _j1i_FST50_0.doc, _j1i_FST50_0.tfp, _j1j.fdt, _j1j.fdx, _j1j.fnm, _j1j.nvd, _j1j.nvm, _j1j.si, _j1j_FST50_0.doc, _j1j_FST50_0.tfp, _j1l.fdt, _j1l.fdx, _j1l.fnm, _j1l.nvd, _j1l.nvm, _j1l.si, _j1l_FST50_0.doc, _j1l_FST50_0.tfp, _j1m.fdt, _j1m.fdx, _j1m.fnm, _j1m.nvd, _j1m.nvm, _j1m.si, _j1m_FST50_0.doc, _j1m_FST50_0.tfp, _j1n.fdt, _j1n.fdx, _j1n.fnm, _j1n.nvd, _j1n.nvm, _j1n.si, _j1n_FST50_0.doc, _j1n_FST50_0.tfp, _j1o.fdt, _j1o.fdx, _j1o.fnm, _j1o.nvd, _j1o.nvm, _j1o.si, _j1o_FST50_0.doc, _j1o_FST50_0.tfp, _j1p.fdt, _j1p.fdx, _j1p.fnm, _j1p.nvd, _j1p.nvm, _j1p.si, _j1p_FST50_0.doc, _j1p_FST50_0.tfp, _j1q.fdt, _j1q.fdx, _j1q.fnm, _j1q.nvd, _j1q.nvm, _j1q.si, _j1q_FST50_0.doc, _j1q_FST50_0.tfp, _j1r.fdt, _j1r.fdx, _j1r.fnm, _j1r.nvd, _j1r.nvm, _j1r.si, _j1r_FST50_0.doc, _j1r_FST50_0.tfp, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, segments_2]}]> but was:<[{indexVersion=1507625043838,generation=2,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j17.cfe, _j17.cfs, _j17.si, _j18.fdt, _j18.fdx, _j18.fnm, _j18.nvd, _j18.nvm, _j18.si, _j18_FST50_0.doc, _j18_FST50_0.tfp, _j19.fdt, _j19.fdx, _j19.fnm, _j19.nvd, _j19.nvm, _j19.si, _j19_FST50_0.doc, _j19_FST50_0.tfp, _j1a.fdt, _j1a.fdx, _j1a.fnm, _j1a.nvd, _j1a.nvm, _j1a.si, _j1a_FST50_0.doc, _j1a_FST50_0.tfp, _j1b.fdt, _j1b.fdx, _j1b.fnm, _j1b.nvd, _j1b.nvm, _j1b.si, _j1b_FST50_0.doc, _j1b_FST50_0.tfp, _j1c.fdt, _j1c.fdx, _j1c.fnm, _j1c.nvd, _j1c.nvm, _j1c.si, _j1c_FST50_0.doc, _j1c_FST50_0.tfp, _j1d.fdt, _j1d.fdx, _j1d.fnm, _j1d.nvd, _j1d.nvm, _j1d.si, _j1d_FST50_0.doc, _j1d_FST50_0.tfp, _j1e.fdt, _j1e.fdx, _j1e.fnm, _j1e.nvd, _j1e.nvm, _j1e.si, _j1e_FST50_0.doc, _j1e_FST50_0.tfp, _j1f.fdt, _j1f.fdx, _j1f.fnm, _j1f.nvd, _j1f.nvm, _j1f.si, _j1f_FST50_0.doc, _j1f_FST50_0.tfp, _j1g.fdt, _j1g.fdx, _j1g.fnm, _j1g.nvd, _j1g.nvm, _j1g.si, _j1g_FST50_0.doc, _j1g_FST50_0.tfp, _j1h.fdt, _j1h.fdx, _j1h.fnm, _j1h.nvd, _j1h.nvm, _j1h.si, _j1h_FST50_0.doc, _j1h_FST50_0.tfp, _j1i.fdt, _j1i.fdx, _j1i.fnm, _j1i.nvd, _j1i.nvm, _j1i.si, _j1i_FST50_0.doc, _j1i_FST50_0.tfp, _j1j.fdt, _j1j.fdx, _j1j.fnm, _j1j.nvd, _j1j.nvm, _j1j.si, _j1j_FST50_0.doc, _j1j_FST50_0.tfp, _j1l.fdt, _j1l.fdx, _j1l.fnm, _j1l.nvd, _j1l.nvm, _j1l.si, _j1l_FST50_0.doc, _j1l_FST50_0.tfp, _j1m.fdt, _j1m.fdx, _j1m.fnm, _j1m.nvd, _j1m.nvm, _j1m.si, _j1m_FST50_0.doc, _j1m_FST50_0.tfp, _j1n.fdt, _j1n.fdx, _j1n.fnm, _j1n.nvd, _j1n.nvm, _j1n.si, _j1n_FST50_0.doc, _j1n_FST50_0.tfp, _j1o.fdt, _j1o.fdx, _j1o.fnm, _j1o.nvd, _j1o.nvm, _j1o.si, _j1o_FST50_0.doc, _j1o_FST50_0.tfp, _j1p.fdt, _j1p.fdx, _j1p.fnm, _j1p.nvd, _j1p.nvm, _j1p.si, _j1p_FST50_0.doc, _j1p_FST50_0.tfp, _j1q.fdt, _j1q.fdx, _j1q.fnm, _j1q.nvd, _j1q.nvm, _j1q.si, _j1q_FST50_0.doc, _j1q_FST50_0.tfp, _j1r.fdt, _j1r.fdx, _j1r.fnm, _j1r.nvd, _j1r.nvm, _j1r.si, _j1r_FST50_0.doc, _j1r_FST50_0.tfp, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, segments_2]}, {indexVersion=1507625043838,generation=3,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j1k.cfe, _j1k.cfs, _j1k.si, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, _j1u.cfe, _j1u.cfs, _j1u.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1507625043838,generation=2,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j17.cfe, _j17.cfs, _j17.si, _j18.fdt, _j18.fdx, _j18.fnm, _j18.nvd, _j18.nvm, _j18.si, _j18_FST50_0.doc, _j18_FST50_0.tfp, _j19.fdt, _j19.fdx, _j19.fnm, _j19.nvd, _j19.nvm, _j19.si, _j19_FST50_0.doc, _j19_FST50_0.tfp, _j1a.fdt, _j1a.fdx, _j1a.fnm, _j1a.nvd, _j1a.nvm, _j1a.si, _j1a_FST50_0.doc, _j1a_FST50_0.tfp, _j1b.fdt, _j1b.fdx, _j1b.fnm, _j1b.nvd, _j1b.nvm, _j1b.si, _j1b_FST50_0.doc, _j1b_FST50_0.tfp, _j1c.fdt, _j1c.fdx, _j1c.fnm, _j1c.nvd, _j1c.nvm, _j1c.si, _j1c_FST50_0.doc, _j1c_FST50_0.tfp, _j1d.fdt, _j1d.fdx, _j1d.fnm, _j1d.nvd, _j1d.nvm, _j1d.si, _j1d_FST50_0.doc, _j1d_FST50_0.tfp, _j1e.fdt, _j1e.fdx, _j1e.fnm, _j1e.nvd, _j1e.nvm, _j1e.si, _j1e_FST50_0.doc, _j1e_FST50_0.tfp, _j1f.fdt, _j1f.fdx, _j1f.fnm, _j1f.nvd, _j1f.nvm, _j1f.si, _j1f_FST50_0.doc, _j1f_FST50_0.tfp, _j1g.fdt, _j1g.fdx, _j1g.fnm, _j1g.nvd, _j1g.nvm, _j1g.si, _j1g_FST50_0.doc, _j1g_FST50_0.tfp, _j1h.fdt, _j1h.fdx, _j1h.fnm, _j1h.nvd, _j1h.nvm, _j1h.si, _j1h_FST50_0.doc, _j1h_FST50_0.tfp, _j1i.fdt, _j1i.fdx, _j1i.fnm, _j1i.nvd, _j1i.nvm, _j1i.si, _j1i_FST50_0.doc, _j1i_FST50_0.tfp, _j1j.fdt, _j1j.fdx, _j1j.fnm, _j1j.nvd, _j1j.nvm, _j1j.si, _j1j_FST50_0.doc, _j1j_FST50_0.tfp, _j1l.fdt, _j1l.fdx, _j1l.fnm, _j1l.nvd, _j1l.nvm, _j1l.si, _j1l_FST50_0.doc, _j1l_FST50_0.tfp, _j1m.fdt, _j1m.fdx, _j1m.fnm, _j1m.nvd, _j1m.nvm, _j1m.si, _j1m_FST50_0.doc, _j1m_FST50_0.tfp, _j1n.fdt, _j1n.fdx, _j1n.fnm, _j1n.nvd, _j1n.nvm, _j1n.si, _j1n_FST50_0.doc, _j1n_FST50_0.tfp, _j1o.fdt, _j1o.fdx, _j1o.fnm, _j1o.nvd, _j1o.nvm, _j1o.si, _j1o_FST50_0.doc, _j1o_FST50_0.tfp, _j1p.fdt, _j1p.fdx, _j1p.fnm, _j1p.nvd, _j1p.nvm, _j1p.si, _j1p_FST50_0.doc, _j1p_FST50_0.tfp, _j1q.fdt, _j1q.fdx, _j1q.fnm, _j1q.nvd, _j1q.nvm, _j1q.si, _j1q_FST50_0.doc, _j1q_FST50_0.tfp, _j1r.fdt, _j1r.fdx, _j1r.fnm, _j1r.nvd, _j1r.nvm, _j1r.si, _j1r_FST50_0.doc, _j1r_FST50_0.tfp, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, segments_2]}]> but was:<[{indexVersion=1507625043838,generation=2,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j17.cfe, _j17.cfs, _j17.si, _j18.fdt, _j18.fdx, _j18.fnm, _j18.nvd, _j18.nvm, _j18.si, _j18_FST50_0.doc, _j18_FST50_0.tfp, _j19.fdt, _j19.fdx, _j19.fnm, _j19.nvd, _j19.nvm, _j19.si, _j19_FST50_0.doc, _j19_FST50_0.tfp, _j1a.fdt, _j1a.fdx, _j1a.fnm, _j1a.nvd, _j1a.nvm, _j1a.si, _j1a_FST50_0.doc, _j1a_FST50_0.tfp, _j1b.fdt, _j1b.fdx, _j1b.fnm, _j1b.nvd, _j1b.nvm, _j1b.si, _j1b_FST50_0.doc, _j1b_FST50_0.tfp, _j1c.fdt, _j1c.fdx, _j1c.fnm, _j1c.nvd, _j1c.nvm, _j1c.si, _j1c_FST50_0.doc, _j1c_FST50_0.tfp, _j1d.fdt, _j1d.fdx, _j1d.fnm, _j1d.nvd, _j1d.nvm, _j1d.si, _j1d_FST50_0.doc, _j1d_FST50_0.tfp, _j1e.fdt, _j1e.fdx, _j1e.fnm, _j1e.nvd, _j1e.nvm, _j1e.si, _j1e_FST50_0.doc, _j1e_FST50_0.tfp, _j1f.fdt, _j1f.fdx, _j1f.fnm, _j1f.nvd, _j1f.nvm, _j1f.si, _j1f_FST50_0.doc, _j1f_FST50_0.tfp, _j1g.fdt, _j1g.fdx, _j1g.fnm, _j1g.nvd, _j1g.nvm, _j1g.si, _j1g_FST50_0.doc, _j1g_FST50_0.tfp, _j1h.fdt, _j1h.fdx, _j1h.fnm, _j1h.nvd, _j1h.nvm, _j1h.si, _j1h_FST50_0.doc, _j1h_FST50_0.tfp, _j1i.fdt, _j1i.fdx, _j1i.fnm, _j1i.nvd, _j1i.nvm, _j1i.si, _j1i_FST50_0.doc, _j1i_FST50_0.tfp, _j1j.fdt, _j1j.fdx, _j1j.fnm, _j1j.nvd, _j1j.nvm, _j1j.si, _j1j_FST50_0.doc, _j1j_FST50_0.tfp, _j1l.fdt, _j1l.fdx, _j1l.fnm, _j1l.nvd, _j1l.nvm, _j1l.si, _j1l_FST50_0.doc, _j1l_FST50_0.tfp, _j1m.fdt, _j1m.fdx, _j1m.fnm, _j1m.nvd, _j1m.nvm, _j1m.si, _j1m_FST50_0.doc, _j1m_FST50_0.tfp, _j1n.fdt, _j1n.fdx, _j1n.fnm, _j1n.nvd, _j1n.nvm, _j1n.si, _j1n_FST50_0.doc, _j1n_FST50_0.tfp, _j1o.fdt, _j1o.fdx, _j1o.fnm, _j1o.nvd, _j1o.nvm, _j1o.si, _j1o_FST50_0.doc, _j1o_FST50_0.tfp, _j1p.fdt, _j1p.fdx, _j1p.fnm, _j1p.nvd, _j1p.nvm, _j1p.si, _j1p_FST50_0.doc, _j1p_FST50_0.tfp, _j1q.fdt, _j1q.fdx, _j1q.fnm, _j1q.nvd, _j1q.nvm, _j1q.si, _j1q_FST50_0.doc, _j1q_FST50_0.tfp, _j1r.fdt, _j1r.fdx, _j1r.fnm, _j1r.nvd, _j1r.nvm, _j1r.si, _j1r_FST50_0.doc, _j1r_FST50_0.tfp, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, segments_2]}, {indexVersion=1507625043838,generation=3,filelist=[_at6.fdt, _at6.fdx, _at6.fnm, _at6.nvd, _at6.nvm, _at6.si, _at6_FST50_0.doc, _at6_FST50_0.tfp, _g9h.fdt, _g9h.fdx, _g9h.fnm, _g9h.nvd, _g9h.nvm, _g9h.si, _g9h_FST50_0.doc, _g9h_FST50_0.tfp, _hdn.cfe, _hdn.cfs, _hdn.si, _ibm.cfe, _ibm.cfs, _ibm.si, _ikj.cfe, _ikj.cfs, _ikj.si, _isa.cfe, _isa.cfs, _isa.si, _j02.cfe, _j02.cfs, _j02.si, _j1k.cfe, _j1k.cfs, _j1k.si, _j1s.fdt, _j1s.fdx, _j1s.fnm, _j1s.nvd, _j1s.nvm, _j1s.si, _j1s_FST50_0.doc, _j1s_FST50_0.tfp, _j1t.fdt, _j1t.fdx, _j1t.fnm, _j1t.nvd, _j1t.nvm, _j1t.si, _j1t_FST50_0.doc, _j1t_FST50_0.tfp, _j1u.cfe, _j1u.cfs, _j1u.si, segments_3]}]>
        at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E:B62D39464256B27D]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13120 lines...]
   [junit4] Suite: org.apache.solr.security.TestPKIAuthenticationPlugin
   [junit4]   2> 2743772 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.TestPKIAuthenticationPlugin_93FA2276321EBC7E-001/init-core-data-001
   [junit4]   2> 2743772 WARN  (SUITE-TestPKIAuthenticationPlugin-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=351 numCloses=351
   [junit4]   2> 2743779 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2743780 INFO  (SUITE-TestPKIAuthenticationPlugin-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2743823 INFO  (TEST-TestPKIAuthenticationPlugin.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2756579 ERROR (TEST-TestPKIAuthenticationPlugin.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.PKIAuthenticationPlugin Invalid key request timestamp: 1507617583675 , received timestamp: 1507617594925 , TTL: 10000
   [junit4]   2> 2756579 INFO  (TEST-TestPKIAuthenticationPlugin.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestPKIAuthenticationPlugin -Dtests.method=test -Dtests.seed=93FA2276321EBC7E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Europe/Helsinki -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   12.9s J2 | TestPKIAuthenticationPlugin.test <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > at __randomizedtesting.SeedInfo.seed([93FA2276321EBC7E:1BAE1DAC9CE2D186]:0)
   [junit4]    > at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.security.TestPKIAuthenticationPlugin_93FA2276321EBC7E-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=845, maxMBSortInHeap=7.466840311266326, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=Europe/Helsinki
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=204484080,total=429916160
   [junit4]   2> NOTE: All tests run in this JVM: [AssignBackwardCompatibilityTest, HdfsRecoverLeaseTest, InfixSuggestersTest, TestJsonFacets, TestSchemaResource, DeleteReplicaTest, TestUseDocValuesAsStored, TestIntervalFaceting, TestLegacyField, StatsComponentTest, TestPKIAuthenticationPlugin]
   [junit4] Completed [72/744 (1!)] on J2 in 13.01s, 1 test, 1 error <<< FAILURES!

[...truncated 11 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/init-core-data-001
   [junit4]   2> 2093506 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 2093506 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2093565 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2093565 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2093565 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ipe/zp
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 2094182 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 2094285 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2094295 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2094417 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_50359_hdfs____kf9buu/webapp
   [junit4]   2> 2096053 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:50359
   [junit4]   2> 2096651 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2096652 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2096725 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_54320_datanode____.b7ilpj/webapp
   [junit4]   2> 2098749 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54320
   [junit4]   2> 2099176 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2099177 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2099240 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_42812_datanode____.zg4vi6/webapp
   [junit4]   2> 2100787 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:52295) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2101722 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1bf4b08dc0378e: from storage DS-35bb9194-b31d-4abf-97ae-c7937da191ce node DatanodeRegistration(127.0.0.1:40519, datanodeUuid=da769b07-9e1a-4507-b851-e5fb1ae8a811, infoPort=36145, infoSecurePort=0, ipcPort=59985, storageInfo=lv=-56;cid=testClusterID;nsid=797234624;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2101722 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1bf4b08dc0378e: from storage DS-d19c0988-1358-485b-a2cc-e49a34fec353 node DatanodeRegistration(127.0.0.1:40519, datanodeUuid=da769b07-9e1a-4507-b851-e5fb1ae8a811, infoPort=36145, infoSecurePort=0, ipcPort=59985, storageInfo=lv=-56;cid=testClusterID;nsid=797234624;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2104266 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[93FA2276321EBC7E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42812
   [junit4]   2> 2105643 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:52295) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2105761 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1bf4b17d9cfce6: from storage DS-be943057-0370-4255-89f7-32241a66e26c node DatanodeRegistration(127.0.0.1:33907, datanodeUuid=c324c1a7-35fc-4db7-a174-f516404d2e59, infoPort=41605, infoSecurePort=0, ipcPort=35815, storageInfo=lv=-56;cid=testClusterID;nsid=797234624;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2105761 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1bf4b17d9cfce6: from storage DS-696c8a59-0aee-48bc-89c7-b7b180ba1494 node DatanodeRegistration(127.0.0.1:33907, datanodeUuid=c324c1a7-35fc-4db7-a174-f516404d2e59, infoPort=41605, infoSecurePort=0, ipcPort=35815, storageInfo=lv=-56;cid=testClusterID;nsid=797234624;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2106510 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2106545 INFO  (Thread-1075) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2106546 INFO  (Thread-1075) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2106547 ERROR (Thread-1075) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2106652 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.ZkTestServer start zk server on port:43657
   [junit4]   2> 2108148 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2108165 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 2108166 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2108174 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2108185 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2108186 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2108193 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2108201 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2108213 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2108214 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2108225 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2108233 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 2110320 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2110329 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6bd8e7f0{/ipe/zp,null,AVAILABLE}
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1264be3{HTTP/1.1,[http/1.1]}{127.0.0.1:48446}
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.Server Started @2130215ms
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:52295/hdfs__localhost_52295__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001_tempDir-002_control_data, replicaType=NRT, hostContext=/ipe/zp, hostPort=48446, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/control-001/cores}
   [junit4]   2> 2110330 ERROR (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2110330 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-10T06:29:05.710Z
   [junit4]   2> 2110647 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2110647 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/control-001/solr.xml
   [junit4]   2> 2110650 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2110650 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2110651 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2110673 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43657/solr
   [junit4]   2> 2110978 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2110979 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:48446_ipe%2Fzp
   [junit4]   2> 2110996 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.Overseer Overseer (id=98803183906062340-127.0.0.1:48446_ipe%2Fzp-n_0000000000) starting
   [junit4]   2> 2111040 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48446_ipe%2Fzp
   [junit4]   2> 2111150 INFO  (OverseerStateUpdate-98803183906062340-127.0.0.1:48446_ipe%2Fzp-n_0000000000) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2111944 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2112062 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2112062 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2112063 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/control-001/cores
   [junit4]   2> 2112945 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2112946 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43657/solr ready
   [junit4]   2> 2113002 INFO  (qtp1012437845-4650) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:48446_ipe%252Fzp&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2113141 INFO  (OverseerThreadFactory-1571-thread-1-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2113265 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2113265 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2113533 INFO  (zkCallback-1203-thread-1-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2115325 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2115438 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2116073 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2116774 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2116775 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2116854 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:52295/solr_hdfs_home
   [junit4]   2> 2116855 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2116855 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2116855 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:52295/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 2116856 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 2117114 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 2117114 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2117114 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2124691 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2124699 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 2124730 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 2124755 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 2124755 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2124755 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2124816 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2124816 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=12.181640625, floorSegmentMB=1.130859375, forceMergeDeletesPctAllowed=0.8670676191146653, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14294353700137236
   [junit4]   2> 2128440 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33907 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-35bb9194-b31d-4abf-97ae-c7937da191ce:NORMAL:127.0.0.1:40519|RBW], ReplicaUC[[DISK]DS-696c8a59-0aee-48bc-89c7-b7b180ba1494:NORMAL:127.0.0.1:33907|RBW]]} size 72
   [junit4]   2> 2128442 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40519 is added to blk_1073741825_1001 size 72
   [junit4]   2> 2128771 WARN  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2129890 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2129890 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2129890 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 2130093 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 10000ms;
   [junit4]   2> 2130093 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommited for 3000ms;
   [junit4]   2> 2130177 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20505617464355322]
   [junit4]   2> 2133629 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3a9848e3[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2133634 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2133634 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2133650 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2133652 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580850970919698432
   [junit4]   2> 2133751 INFO  (searcherExecutor-1574-thread-1-processing-n:127.0.0.1:48446_ipe%2Fzp x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@3a9848e3[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48446/ipe/zp/control_collection_shard1_replica_n1/
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:48446/ipe/zp/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2133779 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2133791 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48446/ipe/zp/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2133797 INFO  (zkCallback-1203-thread-1-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2133849 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2133850 INFO  (qtp1012437845-4648) [n:127.0.0.1:48446_ipe%2Fzp c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=20584
   [junit4]   2> 2134127 INFO  (qtp1012437845-4650) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2134133 INFO  (zkCallback-1203-thread-1-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2135129 INFO  (qtp1012437845-4650) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:48446_ipe%252Fzp&wt=javabin&version=2} status=0 QTime=22126
   [junit4]   2> 2135228 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2135229 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43657/solr ready
   [junit4]   2> 2135229 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2135230 INFO  (qtp1012437845-4651) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2135242 INFO  (OverseerCollectionConfigSetProcessor-98803183906062340-127.0.0.1:48446_ipe%2Fzp-n_0000000000) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2135242 INFO  (OverseerThreadFactory-1571-thread-2-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2135249 WARN  (OverseerThreadFactory-1571-thread-2-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2135631 INFO  (qtp1012437845-4651) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2135631 INFO  (qtp1012437845-4651) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=401
   [junit4]   2> 2136627 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/shard-1-001 of type NRT
   [junit4]   2> 2136627 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2136710 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1d30dabc{/ipe/zp,null,AVAILABLE}
   [junit4]   2> 2136710 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@395956a7{HTTP/1.1,[http/1.1]}{127.0.0.1:34423}
   [junit4]   2> 2136710 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.Server Started @2156595ms
   [junit4]   2> 2136710 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:52295/hdfs__localhost_52295__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/ipe/zp, hostPort=34423, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/shard-1-001/cores}
   [junit4]   2> 2136710 ERROR (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2136710 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2136711 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2136711 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2136711 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-10T06:29:32.091Z
   [junit4]   2> 2136750 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2136750 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/shard-1-001/solr.xml
   [junit4]   2> 2136774 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2136774 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2136775 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2136790 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43657/solr
   [junit4]   2> 2136869 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2136905 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2136913 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34423_ipe%2Fzp
   [junit4]   2> 2136914 INFO  (zkCallback-1203-thread-2-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2137486 INFO  (zkCallback-1210-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2137486 INFO  (zkCallback-1215-thread-1-processing-n:127.0.0.1:34423_ipe%2Fzp) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2138072 INFO  (OverseerCollectionConfigSetProcessor-98803183906062340-127.0.0.1:48446_ipe%2Fzp-n_0000000000) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2139120 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2139219 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2139220 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2139253 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[93FA2276321EBC7E]) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/shard-1-001/cores
   [junit4]   2> 2139546 INFO  (qtp2113264408-4709) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34423_ipe%252Fzp&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2139551 INFO  (OverseerThreadFactory-1571-thread-3-processing-n:127.0.0.1:48446_ipe%2Fzp) [n:127.0.0.1:48446_ipe%2Fzp    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:34423_ipe%2Fzp for creating new replica
   [junit4]   2> 2139581 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2139581 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2140635 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 2140949 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
   [junit4]   2> 2141382 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2141414 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
   [junit4]   2> 2141414 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e61e72e
   [junit4]   2> 2141415 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:52295/solr_hdfs_home
   [junit4]   2> 2141425 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2141425 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2141425 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_93FA2276321EBC7E-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[hdfs://localhost:52295/solr_hdfs_home/collection1/core_node22/data/]
   [junit4]   2> 2141426 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/collection1/core_node22/data/snapshot_metadata
   [junit4]   2> 2141440 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 2141440 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2141440 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2141849 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2141851 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/collection1/core_node22/data
   [junit4]   2> 2142127 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:52295/solr_hdfs_home/collection1/core_node22/data/index
   [junit4]   2> 2142132 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 2142132 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2142132 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2142236 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2142237 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=12.181640625, floorSegmentMB=1.130859375, forceMergeDeletesPctAllowed=0.8670676191146653, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14294353700137236
   [junit4]   2> 2142649 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33907 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-35bb9194-b31d-4abf-97ae-c7937da191ce:NORMAL:127.0.0.1:40519|RBW], ReplicaUC[[DISK]DS-be943057-0370-4255-89f7-32241a66e26c:NORMAL:127.0.0.1:33907|RBW]]} size 0
   [junit4]   2> 2142729 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40519 is added to blk_1073741826_1002 size 72
   [junit4]   2> 2142830 WARN  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2143675 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2143676 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2143676 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 2143897 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 10000ms;
   [junit4]   2> 2143898 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: if uncommited for 3000ms;
   [junit4]   2> 2143906 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20505617464355322]
   [junit4]   2> 2144037 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@26ac9bf5[collection1_shard1_replica_n21] main]
   [junit4]   2> 2144045 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2144045 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2144045 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2144166 INFO  (searcherExecutor-1585-thread-1-processing-n:127.0.0.1:34423_ipe%2Fzp x:collection1_shard1_replica_n21 s:shard1 c:collection1) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@26ac9bf5[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2144166 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580850981944426496
   [junit4]   2> 2144188 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2144188 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2144188 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34423/ipe/zp/collection1_shard1_replica_n21/
   [junit4]   2> 2144188 INFO  (qtp2113264408-4707) [n:127.0.0.1:34423_ipe%2Fzp c:collection1 s:shard1  x:collection1_

[...truncated too long message...]

lable. Index fetch failed by exception: org.apache.solr.common.SolrException: Server refused connection at: http://127.0.0.1:43834/solr/collection1
   [junit4]   2> 10545498 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 10545498 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@14efa294: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4b7bfba
   [junit4]   2> 10545501 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 10545501 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3d65ea: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@efb5090
   [junit4]   2> 10545535 INFO  (coreCloseExecutor-9176-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@39baeb85
   [junit4]   2> 10545535 INFO  (coreCloseExecutor-9176-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=968551301
   [junit4]   2> 10545535 INFO  (coreCloseExecutor-9176-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4f049585: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@86181b5
   [junit4]   2> 10545611 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e474429{/solr,null,UNAVAILABLE}
   [junit4]   2> 10545635 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@57dcaa35{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 10545635 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1041315117
   [junit4]   2> 10545635 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 10545635 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@174a1c78: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@503d24b3
   [junit4]   2> 10545648 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 10545648 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6851d8c: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4b7bfba
   [junit4]   2> 10545672 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 10545672 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b5a83d9: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@efb5090
   [junit4]   2> 10545707 INFO  (coreCloseExecutor-9178-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2fc2ac4
   [junit4]   2> 10545707 INFO  (coreCloseExecutor-9178-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=50080452
   [junit4]   2> 10545707 INFO  (coreCloseExecutor-9178-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7938aa0b: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@61a4c880
   [junit4]   2> 10545763 INFO  (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[93FA2276321EBC7E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2ef2ab9f{/solr,null,UNAVAILABLE}
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_93FA2276321EBC7E-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name=FST50, id=FST50, newname=FST50}, docValues:{}, maxPointsInLeafNode=2012, maxMBSortInHeap=6.960005382026641, sim=RandomSimilarity(queryNorm=true): {}, locale=pt-PT, timezone=Pacific/Rarotonga
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=96255456,total=517996544
   [junit4]   2> NOTE: All tests run in this JVM: [TestDistribIDF, RuleEngineTest, TestCopyFieldCollectionResource, TestRecoveryHdfs, TestSystemCollAutoCreate, ConcurrentDeleteAndCreateCollectionTest, PKIAuthenticationIntegrationTest, TestJsonRequest, DistribDocExpirationUpdateProcessorTest, HdfsBasicDistributedZkTest, DocumentAnalysisRequestHandlerTest, TestUnifiedSolrHighlighter, TestFieldCache, TestUninvertingReader, SpellCheckCollatorWithCollapseTest, SolrIndexSplitterTest, DefaultValueUpdateProcessorTest, TestConfigSets, TestRemoteStreaming, TestTolerantUpdateProcessorRandomCloud, MetricsConfigTest, TestHighlightDedupGrouping, TestExtendedDismaxParser, TestConfigSetsAPIZkFailure, TestMinMaxOnMultiValuedField, CursorPagingTest, SolrIndexMetricsTest, CdcrUpdateLogTest, TestStressLiveNodes, BigEndianAscendingWordSerializerTest, TestRawResponseWriter, TestCustomDocTransformer, SolrCoreTest, SuggesterWFSTTest, QueryElevationComponentTest, TestSolrDeletionPolicy1, ChaosMonkeyNothingIsSafeTest, AtomicUpdateProcessorFactoryTest, TestSchemaSimilarityResource, BasicDistributedZkTest, ExecutePlanActionTest, TestRangeQuery, BinaryUpdateRequestHandlerTest, WordBreakSolrSpellCheckerTest, TestRandomFaceting, LeaderFailureAfterFreshStartTest, AutoAddReplicasPlanActionTest, PreAnalyzedFieldManagedSchemaCloudTest, HdfsDirectoryTest, DeleteLastCustomShardedReplicaTest, TestNumericTokenStream, TestSolrCloudWithSecureImpersonation, CursorMarkTest, IndexBasedSpellCheckerTest, OverseerStatusTest, DocExpirationUpdateProcessorFactoryTest, DistributedSuggestComponentTest, TestFastOutputStream, TestCrossCoreJoin, CdcrBootstrapTest, TestHdfsCloudBackupRestore, TestStressLucene, TestManagedResourceStorage, TestFieldTypeResource, TestJoin, TestSegmentSorting, TestFaceting, TestLazyCores, TestFunctionQuery, TestReplicationHandler]
   [junit4] Completed [690/744 (4!)] on J0 in 787.60s, 16 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 3036 lines...]
   [junit4] Suite: org.apache.solr.analytics.NoFacetCloudTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analytics/test/J1/temp/solr.analytics.NoFacetCloudTest_F3905631EEDD9F00-001/init-core-data-001
   [junit4]   2> Oct 10, 2017 9:32:07 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=es-PY, timezone=America/Chicago
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=240079088,total=312999936
   [junit4]   2> NOTE: All tests run in this JVM: [ExpressionTest, AbstractAnalyticsStatsTest, RangeFacetTest, NoFacetCloudTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=NoFacetCloudTest -Dtests.seed=F3905631EEDD9F00 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-PY -Dtests.timezone=America/Chicago -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | NoFacetCloudTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Could not find collection:collection1
   [junit4]    > at __randomizedtesting.SeedInfo.seed([F3905631EEDD9F00]:0)
   [junit4]    > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
   [junit4]    > at org.apache.solr.analytics.AbstractAnalyticsStatsCloudTest.setupCluster(AbstractAnalyticsStatsCloudTest.java:76)
   [junit4]    > at org.apache.solr.analytics.NoFacetCloudTest.populate(NoFacetCloudTest.java:62)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [12/17 (1!)] on J1 in 84.74s, 0 tests, 1 failure <<< FAILURES!

[...truncated 44070 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 60 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/60/

12 tests failed.
FAILED:  org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.testTermsEnumVariableWidth

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([9EA1AFB6B4C0022A]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([9EA1AFB6B4C0022A]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:97267862FE4AE834]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:690)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1182)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1168)
        ... 38 more


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:97267862FE4AE834]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:690)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1182)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1168)
        ... 38 more


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testTriggerThrottling

Error Message:
Both triggers should have fired by now

Stack Trace:
java.lang.AssertionError: Both triggers should have fired by now
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:E450EF9D821C665E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testTriggerThrottling(TriggerIntegrationTest.java:194)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:48460_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:48460_solr wasn't created
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:7C8CFB45E834823]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:     1) Thread[id=13020, name=zkCallback-1598-thread-1, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13019, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    4) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    5) Thread[id=10915, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    6) Thread[id=10914, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)    7) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
   1) Thread[id=13020, name=zkCallback-1598-thread-1, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13019, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   4) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   5) Thread[id=10915, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   6) Thread[id=10914, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   7) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=13020, name=zkCallback-1598-thread-1, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=10915, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Throwable.getStackTraceElement(Native Method)         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)         at java.lang.Throwable.printStackTrace(Throwable.java:656)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)    5) Thread[id=10914, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.net.URI$Parser.scanIPv4Address(URI.java:3298)         at java.net.URI$Parser.parseIPv4Address(URI.java:3332)         at java.net.URI$Parser.parseServer(URI.java:3234)         at java.net.URI$Parser.parseAuthority(URI.java:3155)         at java.net.URI$Parser.parseHierarchical(URI.java:3097)         at java.net.URI$Parser.parse(URI.java:3053)         at java.net.URI.<init>(URI.java:588)         at java.net.URI.create(URI.java:850)         at org.apache.http.client.methods.HttpPost.<init>(HttpPost.java:73)         at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:456)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)    6) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=13020, name=zkCallback-1598-thread-1, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=10915, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Throwable.getStackTraceElement(Native Method)
        at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
        at java.lang.Throwable.printStackTrace(Throwable.java:656)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
   5) Thread[id=10914, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.net.URI$Parser.scanIPv4Address(URI.java:3298)
        at java.net.URI$Parser.parseIPv4Address(URI.java:3332)
        at java.net.URI$Parser.parseServer(URI.java:3234)
        at java.net.URI$Parser.parseAuthority(URI.java:3155)
        at java.net.URI$Parser.parseHierarchical(URI.java:3097)
        at java.net.URI$Parser.parse(URI.java:3053)
        at java.net.URI.<init>(URI.java:588)
        at java.net.URI.create(URI.java:850)
        at org.apache.http.client.methods.HttpPost.<init>(HttpPost.java:73)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:456)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   6) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Collection not found: collMinRf_1x3

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: collMinRf_1x3
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:97267862FE4AE834]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
        at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:594)
        at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:211)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:97267862FE4AE834]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 seconds
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC:97267862FE4AE834]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
        at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=1746, name=testExecutor-334-thread-8, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1746, name=testExecutor-334-thread-8, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:58863/yne/z
        at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC]:0)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:58863/yne/z
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
        ... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
        ... 9 more




Build Log:
[...truncated 1574 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat
   [junit4]   2> кст 11, 2017 10:35:34 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=8505, name=SUITE-TestLucene70DocValuesFormat-seed#[9EA1AFB6B4C0022A], state=RUNNABLE, group=TGRP-TestLucene70DocValuesFormat]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=8506, name=TEST-TestLucene70DocValuesFormat.testTermsEnumVariableWidth-seed#[9EA1AFB6B4C0022A], state=RUNNABLE, group=TGRP-TestLucene70DocValuesFormat]
   [junit4]   2>         at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
   [junit4]   2>         at sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:46)
   [junit4]   2>         at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
   [junit4]   2>         at sun.nio.ch.IOUtil.read(IOUtil.java:197)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:159)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:178)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
   [junit4]   2>         at org.apache.lucene.store.DataInput.readVInt(DataInput.java:125)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:221)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readVInt(MockIndexInputWrapper.java:183)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$TermsDict.next(Lucene70DocValuesProducer.java:1015)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$TermsDict.seekExact(Lucene70DocValuesProducer.java:1043)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.assertEquals(TestLucene70DocValuesFormat.java:418)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.doTestTermsEnumRandom(TestLucene70DocValuesFormat.java:354)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.testTermsEnumVariableWidth(TestLucene70DocValuesFormat.java:133)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene70DocValuesFormat -Dtests.method=testTermsEnumVariableWidth -Dtests.seed=9EA1AFB6B4C0022A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=Europe/Podgorica -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   47.7s J1 | TestLucene70DocValuesFormat.testTermsEnumVariableWidth <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([9EA1AFB6B4C0022A]:0)
   [junit4]   2> кст 11, 2017 10:35:35 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> кст 11, 2017 10:35:55 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat:
   [junit4]   2>    1) Thread[id=8506, name=TEST-TestLucene70DocValuesFormat.testTermsEnumVariableWidth-seed#[9EA1AFB6B4C0022A], state=RUNNABLE, group=TGRP-TestLucene70DocValuesFormat]
   [junit4]   2>         at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
   [junit4]   2>         at sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:46)
   [junit4]   2>         at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
   [junit4]   2>         at sun.nio.ch.IOUtil.read(IOUtil.java:197)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:159)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterSeekableByteChannel.read(FilterSeekableByteChannel.java:59)
   [junit4]   2>         at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:178)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
   [junit4]   2>         at org.apache.lucene.store.DataInput.readVInt(DataInput.java:125)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:221)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readVInt(MockIndexInputWrapper.java:183)
   [junit4]   2>         at org.apache.lucene.codecs.blocktreeords.OrdsSegmentTermsEnumFrame.loadBlock(OrdsSegmentTermsEnumFrame.java:172)
   [junit4]   2>         at org.apache.lucene.codecs.blocktreeords.OrdsSegmentTermsEnum.seekCeil(OrdsSegmentTermsEnum.java:674)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.assertEquals(TestLucene70DocValuesFormat.java:435)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.doTestTermsEnumRandom(TestLucene70DocValuesFormat.java:371)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat.testTermsEnumVariableWidth(TestLucene70DocValuesFormat.java:133)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> кст 11, 2017 10:35:55 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8506, name=TEST-TestLucene70DocValuesFormat.testTermsEnumVariableWidth-seed#[9EA1AFB6B4C0022A], state=RUNNABLE, group=TGRP-TestLucene70DocValuesFormat]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.codecs.lucene70.TestLucene70DocValuesFormat_9EA1AFB6B4C0022A-001
   [junit4]   2> Oct 11, 2017 8:35:55 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=110, maxMBSortInHeap=7.768972657828554, sim=RandomSimilarity(queryNorm=true): {fieldname=ClassicSimilarity, id=IB SPL-D1}, locale=be, timezone=Europe/Podgorica
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=161247304,total=338165760
   [junit4]   2> NOTE: All tests run in this JVM: [Test2BDocs, TestIndexWriterUnicode, TestPhraseQuery, TestElevationComparator, TestMergedIterator, TestMaxPosition, TestDocValuesQueries, TestSizeBoundedForceMerge, TestRecyclingByteBlockAllocator, TestDeterminism, TestAtomicUpdate, TestNumericDocValuesUpdates, TestReusableStringReader, TestLucene60FieldInfoFormat, TestMixedDocValuesUpdates, TestByteArrayDataInput, TestComplexExplanations, TestCustomSearcherSort, TestIndexFileDeleter, TestRecyclingIntBlockAllocator, TestReaderClosed, TestTimeLimitingCollector, TestLucene50StoredFieldsFormat, TestGraphTokenStreamFiniteStrings, TestPriorityQueue, TestTransactions, TestDocInverterPerFieldErrorInfo, TestSameScoresWithThreads, TestTrackingDirectoryWrapper, TestBooleanOr, TestTermdocPerf, TestRegexpRandom2, TestAllFilesDetectTruncation, TestCustomTermFreq, TestBytesRefAttImpl, TestPrefixInBooleanQuery, TestSparseFixedBitDocIdSet, TestLevenshteinAutomata, TestCharArraySet, TestSumDocFreq, TestPerFieldDocValuesFormat, TestReqExclBulkScorer, TestParallelReaderEmptyIndex, TestAxiomaticSimilarity, TestDoubleValuesSource, TestSpanCollection, TestCustomNorms, TestClassicSimilarity, TestDirectory, TestNativeFSLockFactory, TestLucene70DocValuesFormat]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene70DocValuesFormat -Dtests.seed=9EA1AFB6B4C0022A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=Europe/Podgorica -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestLucene70DocValuesFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([9EA1AFB6B4C0022A]:0)
   [junit4] Completed [315/455 (1!)] on J1 in 7221.29s, 115 tests, 2 errors <<< FAILURES!

[...truncated 11593 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_1F7247B850B685CC-001/init-core-data-001
   [junit4]   2> 1463239 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1463239 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1463313 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1463313 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1463313 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /yne/z
   [junit4]   2> 1474806 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1483204 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1486199 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 1486521 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1490976 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1491638 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_45245_hdfs____.olts1w/webapp
   [junit4]   2> 1503685 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45245
   [junit4]   2> 1517477 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1517497 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1517815 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_43646_datanode____3tadhc/webapp
   [junit4]   2> 1520046 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43646
   [junit4]   2> 1525721 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1525722 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1526289 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_49743_datanode____.7cuqmg/webapp
   [junit4]   2> 1530009 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[1F7247B850B685CC]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49743
   [junit4]   2> 1539752 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_1F7247B850B685CC-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_1F7247B850B685CC-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:50623) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1540048 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_1F7247B850B685CC-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_1F7247B850B685CC-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:50623) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1542730 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1c8c82db076b5f: from storage DS-f1d4c8da-232c-4bd3-9375-b6e4f1793181 node DatanodeRegistration(127.0.0.1:41450, datanodeUuid=f6894042-6d48-465f-be93-b0865422df2c, infoPort=44431, infoSecurePort=0, ipcPort=41637, storageInfo=lv=-56;cid=testClusterID;nsid=1413512869;c=0), blocks: 0, hasStaleStorage: true, processing time: 118 msecs
   [junit4]   2> 1542731 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1c8c82da289f61: from storage DS-514d5007-e979-4acf-8c1b-a25caecdec66 node DatanodeRegistration(127.0.0.1:53101, datanodeUuid=141084dc-2a68-45e3-8e84-2bc440c864d0, infoPort=39803, infoSecurePort=0, ipcPort=39145, storageInfo=lv=-56;cid=testClusterID;nsid=1413512869;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 1542731 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1c8c82db076b5f: from storage DS-014713ce-1e13-40db-9ce5-e83bff2f9975 node DatanodeRegistration(127.0.0.1:41450, datanodeUuid=f6894042-6d48-465f-be93-b0865422df2c, infoPort=44431, infoSecurePort=0, ipcPort=41637, storageInfo=lv=-56;cid=testClusterID;nsid=1413512869;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1542731 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1c8c82da289f61: from storage DS-38ce2fb1-bfef-454f-a121-a47bf61ceae8 node DatanodeRegistration(127.0.0.1:53101, datanodeUuid=141084dc-2a68-45e3-8e84-2bc440c864d0, infoPort=39803, infoSecurePort=0, ipcPort=39145, storageInfo=lv=-56;cid=testClusterID;nsid=1413512869;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1555543 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1555585 INFO  (Thread-118) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1555585 INFO  (Thread-118) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1556277 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.ZkTestServer start zk server on port:57946
   [junit4]   2> 1556916 ERROR (Thread-118) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1557680 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f0eec300b0000, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1557910 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1557997 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1558093 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1558347 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1558445 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1558466 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1558467 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1558485 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1558487 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1558525 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1558527 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1558585 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 1562974 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1563697 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@747442fd{/yne/z,null,AVAILABLE}
   [junit4]   2> 1563875 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@356f7d39{HTTP/1.1,[http/1.1]}{127.0.0.1:56199}
   [junit4]   2> 1563876 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.e.j.s.Server Started @1585286ms
   [junit4]   2> 1563876 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[1F7247B850B685CC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhos

[...truncated too long message...]

va.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=13019, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   [junit4]    >    4) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    5) Thread[id=10915, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   [junit4]    >    6) Thread[id=10914, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   [junit4]    >    7) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=13020, name=zkCallback-1598-thread-1, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=13717, name=zkCallback-1598-thread-3, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=13018, name=StoppableIndexingThread-SendThread(127.0.0.1:40332), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    4) Thread[id=10915, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]    >         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:656)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:643)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:634)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
   [junit4]    >    5) Thread[id=10914, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at java.net.URI$Parser.scanIPv4Address(URI.java:3298)
   [junit4]    >         at java.net.URI$Parser.parseIPv4Address(URI.java:3332)
   [junit4]    >         at java.net.URI$Parser.parseServer(URI.java:3234)
   [junit4]    >         at java.net.URI$Parser.parseAuthority(URI.java:3155)
   [junit4]    >         at java.net.URI$Parser.parseHierarchical(URI.java:3097)
   [junit4]    >         at java.net.URI$Parser.parse(URI.java:3053)
   [junit4]    >         at java.net.URI.<init>(URI.java:588)
   [junit4]    >         at java.net.URI.create(URI.java:850)
   [junit4]    >         at org.apache.http.client.methods.HttpPost.<init>(HttpPost.java:73)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:456)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
   [junit4]    >         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]    >         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]    >         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]    >    6) Thread[id=13716, name=zkCallback-1598-thread-2, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([1F7247B850B685CC]:0)
   [junit4] Completed [320/744 (7!)] on J2 in 261.05s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 48200 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 61 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/61/

10 tests failed.
FAILED:  org.apache.lucene.codecs.memory.TestMemoryDocValuesFormat.testBinaryFixedLengthVsStoredFields

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([6CB5EC62AC1F849B]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.codecs.memory.TestMemoryDocValuesFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([6CB5EC62AC1F849B]:0)


FAILED:  org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([B214C1465D2D4E3]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterOnVMError

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([B214C1465D2D4E3]:0)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A:468FA2487BEF0D92]:0)
        at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1323)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestConfigReload

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.handler.TestConfigReload:     1) Thread[id=11994, name=zkCallback-1410-thread-1, state=TIMED_WAITING, group=TGRP-TestConfigReload]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestConfigReload:
   1) Thread[id=11994, name=zkCallback-1410-thread-1, state=TIMED_WAITING, group=TGRP-TestConfigReload]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestConfigReload

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=11994, name=zkCallback-1410-thread-1, state=TIMED_WAITING, group=TGRP-TestConfigReload]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=11994, name=zkCallback-1410-thread-1, state=TIMED_WAITING, group=TGRP-TestConfigReload]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A]:0)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:60529_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:60529_solr wasn't created
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A:D661159EDB26AD85]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node4":{             "core":"collMinRf_1x3_shard1_replica_n2",             "base_url":"http://127.0.0.1:35739",             "node_name":"127.0.0.1:35739_",             "state":"active",             "type":"NRT"},           "core_node5":{             "core":"collMinRf_1x3_shard1_replica_n3",             "base_url":"http://127.0.0.1:56687",             "node_name":"127.0.0.1:56687_",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node6":{             "core":"collMinRf_1x3_shard1_replica_n1",             "base_url":"http://127.0.0.1:52172",             "node_name":"127.0.0.1:52172_",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"3",     "tlogReplicas":"0"},   "collection1":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node44":{             "core":"collection1_shard1_replica_n43",             "base_url":"http://127.0.0.1:53968",             "node_name":"127.0.0.1:53968_",             "state":"active",             "type":"NRT",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node42":{             "core":"collection1_shard2_replica_n41",             "base_url":"http://127.0.0.1:56687",             "node_name":"127.0.0.1:56687_",             "state":"active",             "type":"NRT",             "leader":"true"},           "core_node46":{             "core":"collection1_shard2_replica_n45",             "base_url":"http://127.0.0.1:35739",             "node_name":"127.0.0.1:35739_",             "state":"active",             "type":"NRT"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"control_collection_shard1_replica_n1",             "base_url":"http://127.0.0.1:52172",             "node_name":"127.0.0.1:52172_",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   "c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node3":{             "state":"down",             "base_url":"http://127.0.0.1:53968",             "core":"c8n_1x2_shard1_replica_n1",             "node_name":"127.0.0.1:53968_",             "type":"NRT"},           "core_node4":{             "core":"c8n_1x2_shard1_replica_n2",             "base_url":"http://127.0.0.1:56687",             "node_name":"127.0.0.1:56687_",             "state":"active",             "type":"NRT",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"2",     "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collMinRf_1x3_shard1_replica_n2",
            "base_url":"http://127.0.0.1:35739",
            "node_name":"127.0.0.1:35739_",
            "state":"active",
            "type":"NRT"},
          "core_node5":{
            "core":"collMinRf_1x3_shard1_replica_n3",
            "base_url":"http://127.0.0.1:56687",
            "node_name":"127.0.0.1:56687_",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node6":{
            "core":"collMinRf_1x3_shard1_replica_n1",
            "base_url":"http://127.0.0.1:52172",
            "node_name":"127.0.0.1:52172_",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node44":{
            "core":"collection1_shard1_replica_n43",
            "base_url":"http://127.0.0.1:53968",
            "node_name":"127.0.0.1:53968_",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node42":{
            "core":"collection1_shard2_replica_n41",
            "base_url":"http://127.0.0.1:56687",
            "node_name":"127.0.0.1:56687_",
            "state":"active",
            "type":"NRT",
            "leader":"true"},
          "core_node46":{
            "core":"collection1_shard2_replica_n45",
            "base_url":"http://127.0.0.1:35739",
            "node_name":"127.0.0.1:35739_",
            "state":"active",
            "type":"NRT"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:52172",
            "node_name":"127.0.0.1:52172_",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "state":"down",
            "base_url":"http://127.0.0.1:53968",
            "core":"c8n_1x2_shard1_replica_n1",
            "node_name":"127.0.0.1:53968_",
            "type":"NRT"},
          "core_node4":{
            "core":"c8n_1x2_shard1_replica_n2",
            "base_url":"http://127.0.0.1:56687",
            "node_name":"127.0.0.1:56687_",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"2",
    "tlogReplicas":"0"}}
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A:468FA2487BEF0D92]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
        at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:370)
        at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 seconds
        at __randomizedtesting.SeedInfo.seed([CEDB9D92D513606A:468FA2487BEF0D92]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
        at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1650 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2> Oct 13, 2017 12:43:54 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=2447, name=SUITE-TestIndexWriterOnVMError-seed#[B214C1465D2D4E3], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    4) Thread[id=2448, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[B214C1465D2D4E3], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:607)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
   [junit4]   2>         at org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:221)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:247)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:131)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:658)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3201)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3413)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3378)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.method=testCheckpoint -Dtests.seed=B214C1465D2D4E3 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-CA -Dtests.timezone=AGT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J0 | TestIndexWriterOnVMError.testCheckpoint <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B214C1465D2D4E3]:0)
   [junit4]   2> Oct 13, 2017 12:43:55 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Oct 13, 2017 12:44:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterOnVMError:
   [junit4]   2>    1) Thread[id=2448, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[B214C1465D2D4E3], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addTermsDict(Lucene70DocValuesConsumer.java:465)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:627)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
   [junit4]   2>         at org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:221)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:247)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:131)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:658)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:453)
   [junit4]   2>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:175)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Oct 13, 2017 12:44:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=2448, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[B214C1465D2D4E3], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2> Oct 13, 2017 12:44:19 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=2448, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[B214C1465D2D4E3], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:71)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:590)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
   [junit4]   2>         at org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:221)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:247)
   [junit4]   2>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:131)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:658)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3201)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3413)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3378)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {text_payloads=BlockTreeOrds(blocksize=128), text_vectors=PostingsFormat(name=Asserting), text1=PostingsFormat(name=Direct), id=Lucene50(blocksize=128)}, docValues:{dv3=DocValuesFormat(name=Direct), dv2=DocValuesFormat(name=Asserting), dv5=DocValuesFormat(name=Lucene70), dv=DocValuesFormat(name=Lucene70), text_payloads=DocValuesFormat(name=Direct), dv4=DocValuesFormat(name=Lucene70), text_vectors=DocValuesFormat(name=Lucene70), text1=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene70), point2d=DocValuesFormat(name=Lucene70), stored1=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1892, maxMBSortInHeap=7.141004137114521, sim=RandomSimilarity(queryNorm=false): {text_payloads=DFR I(F)L2, text_vectors=DFR GB2, text1=IB SPL-D2}, locale=en-CA, timezone=AGT
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=143495048,total=526909440
   [junit4]   2> NOTE: All tests run in this JVM: [TestPagedBytes, TestPerFieldPostingsFormat, TestSloppyMath, TestGeoUtils, TestDeterminizeLexicon, TestMaxTermFrequency, TestSimilarity, TestTermsEnum, TestBagOfPositions, TestIndexingSequenceNumbers, TestSort, TestSearchAfter, TestStandardAnalyzer, TestSearch, Test2BTerms, TestTotalHitCountCollector, TestCodecs, TestBlockPostingsFormat2, TestManyFields, TestThreadedForceMerge, TestRegexpRandom, TestCrashCausesCorruptIndex, TestIntBlockPool, TestCharFilter, TestNIOFSDirectory, TestSnapshotDeletionPolicy, TestBytesRef, TestIndexWriterOnVMError]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.seed=B214C1465D2D4E3 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-CA -Dtests.timezone=AGT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestIndexWriterOnVMError (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([B214C1465D2D4E3]:0)
   [junit4] Completed [311/455 (1!)] on J0 in 7225.17s, 1 test, 2 errors <<< FAILURES!

[...truncated 5776 lines...]
   [junit4] Suite: org.apache.lucene.codecs.memory.TestMemoryDocValuesFormat
   [junit4]   2> 10? 13, 2017 9:42:52 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.codecs.memory.TestMemoryDocValuesFormat
   [junit4]   2>    1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=255, name=SUITE-TestMemoryDocValuesFormat-seed#[6CB5EC62AC1F849B], state=RUNNABLE, group=TGRP-TestMemoryDocValuesFormat]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    4) Thread[id=256, name=TEST-TestMemoryDocValuesFormat.testBinaryFixedLengthVsStoredFields-seed#[6CB5EC62AC1F849B], state=TIMED_WAITING, group=TGRP-TestMemoryDocValuesFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4628)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2091)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2019)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:429)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryVsStoredFields(BaseDocValuesFormatTestCase.java:1472)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1508)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.testBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1497)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestMemoryDocValuesFormat -Dtests.method=testBinaryFixedLengthVsStoredFields -Dtests.seed=6CB5EC62AC1F849B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Atlantic/Madeira -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7094s J0 | TestMemoryDocValuesFormat.testBinaryFixedLengthVsStoredFields <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > at __randomizedtesting.SeedInfo.seed([6CB5EC62AC1F849B]:0)
   [junit4]   2> 10? 13, 2017 9:42:52 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 10? 13, 2017 9:43:12 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.codecs.memory.TestMemoryDocValuesFormat:
   [junit4]   2>    1) Thread[id=256, name=TEST-TestMemoryDocValuesFormat.testBinaryFixedLengthVsStoredFields-seed#[6CB5EC62AC1F849B], state=TIMED_WAITING, group=TGRP-TestMemoryDocValuesFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4628)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2091)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2019)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:429)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryVsStoredFields(BaseDocValuesFormatTestCase.java:1472)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1508)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.testBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1497)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 10? 13, 2017 9:43:12 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=256, name=TEST-TestMemoryDocValuesFormat.testBinaryFixedLengthVsStoredFields-seed#[6CB5EC62AC1F849B], state=TIMED_WAITING, group=TGRP-TestMemoryDocValuesFormat]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/codecs/test/J0/temp/lucene.codecs.memory.TestMemoryDocValuesFormat_6CB5EC62AC1F849B-001
   [junit4]   2> Oct 13, 2017 8:43:12 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=628, maxMBSortInHeap=6.493987228103301, sim=RandomSimilarity(queryNorm=true): {fieldname=LM Jelinek-Mercer(0.100000)}, locale=ko-KR, timezone=Atlantic/Madeira
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=224596248,total=408944640
   [junit4]   2> NOTE: All tests run in this JVM: [TestSimpleTextTermVectorsFormat, TestSimpleTextStoredFieldsFormat, TestMemoryDocValuesFormat]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestMemoryDocValuesFormat -Dtests.seed=6CB5EC62AC1F849B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Atlantic/Madeira -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestMemoryDocValuesFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > at __randomizedtesting.SeedInfo.seed([6CB5EC62AC1F849B]:0)
   [junit4] Completed [20/20 (1!)] on J0 in 7220.43s, 6 tests, 2 errors <<< FAILURES!

[...truncated 6044 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/init-core-data-001
   [junit4]   2> 4736900 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=186 numCloses=186
   [junit4]   2> 4736901 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 4736902 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 4736902 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4736902 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4737598 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 4737759 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4737838 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4737981 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_39195_hdfs____294p75/webapp
   [junit4]   2> 4742637 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39195
   [junit4]   2> 4744923 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4744924 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4744975 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60271_datanode____.qn6hs7/webapp
   [junit4]   2> 4747864 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60271
   [junit4]   2> 4748939 WARN  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4748940 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4749004 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37029_datanode____dbnr3u/webapp
   [junit4]   2> 4750765 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:50925) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4751487 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1d1d7fe9a60803: from storage DS-385cf686-25f4-49d1-94b5-8992fcbac21a node DatanodeRegistration(127.0.0.1:57070, datanodeUuid=b8a78467-605a-4250-b1d3-d45d893495e6, infoPort=51939, infoSecurePort=0, ipcPort=55452, storageInfo=lv=-56;cid=testClusterID;nsid=797534868;c=0), blocks: 0, hasStaleStorage: true, processing time: 2 msecs
   [junit4]   2> 4751487 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1d1d7fe9a60803: from storage DS-9b7d8b72-de5d-4977-b42f-fc441d26353a node DatanodeRegistration(127.0.0.1:57070, datanodeUuid=b8a78467-605a-4250-b1d3-d45d893495e6, infoPort=51939, infoSecurePort=0, ipcPort=55452, storageInfo=lv=-56;cid=testClusterID;nsid=797534868;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4754307 INFO  (SUITE-HdfsTlogReplayBufferedWhileIndexingTest-seed#[CEDB9D92D513606A]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37029
   [junit4]   2> 4761056 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:50925) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4761277 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1d1d822f245e36: from storage DS-1784222d-50a1-4624-a7f6-636be5ca11a6 node DatanodeRegistration(127.0.0.1:41026, datanodeUuid=35111e7b-ac8b-456f-8d96-c1e943e88aae, infoPort=48390, infoSecurePort=0, ipcPort=35882, storageInfo=lv=-56;cid=testClusterID;nsid=797534868;c=0), blocks: 0, hasStaleStorage: true, processing time: 40 msecs
   [junit4]   2> 4761293 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x1d1d822f245e36: from storage DS-75b9b180-d5e7-4a9a-85db-060ac81cb0f7 node DatanodeRegistration(127.0.0.1:41026, datanodeUuid=35111e7b-ac8b-456f-8d96-c1e943e88aae, infoPort=48390, infoSecurePort=0, ipcPort=35882, storageInfo=lv=-56;cid=testClusterID;nsid=797534868;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4761627 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4761627 INFO  (Thread-7136) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4761627 INFO  (Thread-7136) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4761671 ERROR (Thread-7136) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4761734 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.ZkTestServer start zk server on port:43730
   [junit4]   2> 4762503 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4762504 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 4762505 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4762505 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4762523 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4762524 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4762525 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4762525 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4762543 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4762544 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4762544 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4762584 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4763765 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 4763786 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24da635f{/,null,AVAILABLE}
   [junit4]   2> 4763786 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4f6094ec{HTTP/1.1,[http/1.1]}{127.0.0.1:59235}
   [junit4]   2> 4763786 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.e.j.s.Server Started @4782523ms
   [junit4]   2> 4763786 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:50925/hdfs__localhost_50925__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=59235, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest_CEDB9D92D513606A-001/control-001/cores}
   [junit4]   2> 4763787 ERROR (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4763787 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 4763787 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4763787 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 4763787 INFO  (TEST-HdfsTlogReplayBufferedWhileIndexingTest.test-seed#[CEDB9D92D513606A]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-14T01:08:15.

[...truncated too long message...]

leader, tag=1437374817
   [junit4]   2> 9391060 INFO  (jetty-closer-3680-thread-1) [    ] o.a.s.c.Overseer Overseer (id=98824866184232966-127.0.0.1:54042_solr-n_0000000001) closing
   [junit4]   2> 9391060 INFO  (OverseerStateUpdate-98824866184232966-127.0.0.1:54042_solr-n_0000000001) [n:127.0.0.1:54042_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54042_solr
   [junit4]   2> 9391060 WARN  (OverseerAutoScalingTriggerThread-98824866184232966-127.0.0.1:54042_solr-n_0000000001) [n:127.0.0.1:54042_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 9391073 WARN  (zkCallback-3691-thread-5-processing-n:127.0.0.1:54042_solr) [n:127.0.0.1:54042_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 9391074 INFO  (jetty-closer-3680-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@231de5c6{/solr,null,UNAVAILABLE}
   [junit4]   2> 9391092 INFO  (zkCallback-3702-thread-4-processing-n:127.0.0.1:57668_solr) [n:127.0.0.1:57668_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 9391092 INFO  (zkCallback-3702-thread-5-processing-n:127.0.0.1:57668_solr) [n:127.0.0.1:57668_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57668_solr
   [junit4]   2> 9395821 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
   [junit4]   2> 9395821 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
   [junit4]   2> 9395821 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.c.SolrCore [recoverytest_shard1_replica_n3]  CLOSING SolrCore org.apache.solr.core.SolrCore@40f1dba6
   [junit4]   2> 9395821 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.recoverytest.shard1.replica_n3, tag=1089592230
   [junit4]   2> 9395821 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6113cb3a: rootName = solr_57668, domain = solr.core.recoverytest.shard1.replica_n3, service url = null, agent id = null] for registry solr.core.recoverytest.shard1.replica_n3 / com.codahale.metrics.MetricRegistry@1fc2f37a
   [junit4]   2> 9395828 INFO  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.recoverytest.shard1.leader, tag=1089592230
   [junit4]   2> 9395829 WARN  (recoveryExecutor-3700-thread-1-processing-n:127.0.0.1:57668_solr x:recoverytest_shard1_replica_n3 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.c.RecoveryStrategy Stopping recovery for core=[recoverytest_shard1_replica_n3] coreNodeName=[core_node4]
   [junit4]   2> 9395830 WARN  (updateExecutor-3699-thread-2-processing-n:127.0.0.1:57668_solr) [n:127.0.0.1:57668_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n3] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
   [junit4]   2> 9395834 WARN  (zkCallback-3702-thread-5-processing-n:127.0.0.1:57668_solr) [n:127.0.0.1:57668_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 9395834 INFO  (jetty-closer-3680-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2ad256f6{/solr,null,UNAVAILABLE}
   [junit4]   2> 9395840 ERROR (SUITE-RecoveryZkTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 9395840 INFO  (SUITE-RecoveryZkTest-seed#[CEDB9D92D513606A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53106 53106
   [junit4]   2> 9395932 INFO  (Thread-13460) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53106 53106
   [junit4]   2> 9395932 WARN  (Thread-13460) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 4 /solr/aliases.json
   [junit4]   2> 4 /solr/clusterprops.json
   [junit4]   2> 3 /solr/security.json
   [junit4]   2> 3 /solr/configs/conf
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 28 /solr/collections/recoverytest/state.json
   [junit4]   2> 4 /solr/clusterstate.json
   [junit4]   2> 2 /solr/collections/recoverytest/leader_elect/shard1/election/98824866184232966-core_node2-n_0000000000
   [junit4]   2> 2 /solr/autoscaling.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 4 /solr/collections
   [junit4]   2> 3 /solr/live_nodes
   [junit4]   2> 2 /solr/overseer/queue
   [junit4]   2> 2 /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 2 /solr/overseer/collection-queue-work
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_CEDB9D92D513606A-001
   [junit4]   2> Oct 14, 2017 2:25:30 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=PostingsFormat(name=Memory), a_t=Lucene50(blocksize=128), a_i=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1961, maxMBSortInHeap=6.275634190549594, sim=RandomSimilarity(queryNorm=false): {}, locale=he, timezone=America/Belem
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=116861792,total=524812288
   [junit4]   2> NOTE: All tests run in this JVM: [TestSurroundQueryParser, ChaosMonkeyNothingIsSafeWithPullReplicasTest, OverseerTest, AssignBackwardCompatibilityTest, ComputePlanActionTest, FullSolrCloudDistribCmdsTest, TestCloudSchemaless, NotRequiredUniqueKeyTest, DistributedFacetPivotLargeTest, ConnectionManagerTest, HttpTriggerListenerTest, ReplaceNodeTest, ParsingFieldUpdateProcessorsTest, TestDelegationWithHadoopAuth, TestIndexSearcher, ShowFileRequestHandlerTest, UniqFieldsUpdateProcessorFactoryTest, ConfigureRecoveryStrategyTest, TermVectorComponentDistributedTest, TestQuerySenderListener, AnalyticsQueryTest, ImplicitSnitchTest, TestReversedWildcardFilterFactory, JvmMetricsTest, MultiThreadedOCPTest, V2StandaloneTest, TestDynamicFieldResource, TestSortByMinMaxFunction, BasicDistributedZk2Test, SolrJmxReporterTest, BadIndexSchemaTest, TestTolerantSearch, MetricsHandlerTest, TestLockTree, AddSchemaFieldsUpdateProcessorFactoryTest, CleanupOldIndexTest, TemplateUpdateProcessorTest, TestHighFrequencyDictionaryFactory, TestSQLHandler, TestSearcherReuse, TestClusterStateMutator, HLLUtilTest, TestQueryUtils, TestSolrConfigHandler, TestZkChroot, TestCollectionAPIs, BasicAuthStandaloneTest, TestSolrCloudWithKerberosAlt, TestReloadAndDeleteDocs, MultiTermTest, TermsComponentTest, TestFoldingMultitermQuery, TestPointFields, PingRequestHandlerTest, DirectUpdateHandlerTest, InfixSuggestersTest, ClassificationUpdateProcessorFactoryTest, NumericFieldsTest, TestFilteredDocIdSet, TestInPlaceUpdatesStandalone, DateMathParserTest, TestStressReorder, DistributedDebugComponentTest, TestRetrieveFieldsOptimizer, BigEndianAscendingWordDeserializerTest, SearchHandlerTest, HardAutoCommitTest, ConnectionReuseTest, TestInfoStreamLogging, HighlighterMaxOffsetTest, TestPushWriter, TestStandardQParsers, DateRangeFieldTest, TestConfigSetsAPIExclusivity, PeerSyncReplicationTest, SolrTestCaseJ4Test, ManagedSchemaRoundRobinCloudTest, SolrCoreCheckLockOnStartupTest, BlockJoinFacetRandomTest, TriLevelCompositeIdRoutingTest, TestCloudRecovery, TestDocTermOrds, TestReload, TestComplexPhraseQParserPlugin, XsltUpdateRequestHandlerTest, TestZkAclsWithHadoopAuth, TestJsonFacets, UUIDUpdateProcessorFallbackTest, TestSort, StatsComponentTest, TestExactStatsCache, LeaderElectionTest, TestRandomFlRTGCloud, TestExceedMaxTermLength, OverseerRolesTest, SimplePostToolTest, HdfsChaosMonkeySafeLeaderTest, TestTestInjection, MinimalSchemaTest, HdfsUnloadDistributedZkTest, TestLegacyField, HdfsRecoverLeaseTest, RecoveryZkTest]
   [junit4] Completed [724/744 (5!)] on J2 in 143.87s, 1 test, 1 error <<< FAILURES!

[...truncated 47003 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 62 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/62/

14 tests failed.
FAILED:  org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([3D7A7B2B0644DCE]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestInetAddressRangeQueries

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([3D7A7B2B0644DCE]:0)


FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([2611DEC7FC02C210]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([2611DEC7FC02C210]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:BB3B716A4F1DF44B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:BB3B716A4F1DF44B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:38956_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:38956_solr wasn't created
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:2BD5C6BCEFD4545C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:34511/sjoho/j/collection1: non ok status: 500, message:Server Error

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34511/sjoho/j/collection1: non ok status: 500, message:Server Error
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:BB3B716A4F1DF44B]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:556)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at org.apache.solr.BaseDistributedSearchTestCase.add(BaseDistributedSearchTestCase.java:507)
        at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:683)
        at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:377)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
Index: 0, Size: 0

Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:272715E5C2E624AD]:0)
        at java.util.ArrayList.rangeCheck(ArrayList.java:653)
        at java.util.ArrayList.get(ArrayList.java:429)
        at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication

Error Message:
Shard shard1_0 replicas do not have same number of documents expected:<18> but was:<0>

Stack Trace:
java.lang.AssertionError: Shard shard1_0 replicas do not have same number of documents expected:<18> but was:<0>
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:7925DAB07049B6B6]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.ShardSplitTest.assertConsistentReplicas(ShardSplitTest.java:243)
        at org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication(ShardSplitTest.java:217)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:CBB1DB58998D48E0]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:68)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=2=count(//lst[@name='segments']/lst/str[@name='version'][.='7.2.0'])
        xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><lst name="segments"><lst name="_0"><str name="name">_0</str><int name="delCount">1</int><long name="sizeInBytes">1331</long><int name="size">4</int><date name="age">2017-10-16T00:29:17.453Z</date><str name="source">flush</str><str name="version">7.2.0</str></lst><lst name="_1"><str name="name">_1</str><int name="delCount">0</int><long name="sizeInBytes">1199</long><int name="size">1</int><date name="age">2017-10-16T00:29:17.488Z</date><str name="source">flush</str><str name="version">7.2.0</str></lst><lst name="_2"><str name="name">_2</str><int name="delCount">0</int><long name="sizeInBytes">1264</long><int name="size">4</int><date name="age">2017-10-16T00:29:17.509Z</date><str name="source">flush</str><str name="version">7.2.0</str></lst><lst name="_3"><str name="name">_3</str><int name="delCount">0</int><long name="sizeInBytes">1199</long><int name="size">1</int><date name="age">2017-10-16T00:29:17.510Z</date><str name="source">flush</str><str name="version">7.2.0</str></lst></lst>
</response>

        request was:qt=/admin/segments&wt=xml
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
        ... 40 more


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:BB3B716A4F1DF44B]:0)
        at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestAuthenticationFramework.testBasics

Error Message:
Error from server at http://127.0.0.1:40461/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:40461/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([336F4EB0E1E199B3:EB7E09CD90FC7C3]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
        at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40461/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/29)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node3":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node3/data/",           "base_url":"http://127.0.0.1:45178",           "node_name":"127.0.0.1:45178_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",           "core":"solrj_collection4_shard1_replica_n1",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node7/data/",           "base_url":"http://127.0.0.1:37317",           "node_name":"127.0.0.1:37317_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",           "core":"solrj_collection4_shard2_replica_n2",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node10/data/",           "base_url":"http://127.0.0.1:41412",           "node_name":"127.0.0.1:41412_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node10/data/tlog",           "core":"solrj_collection4_shard3_replica_n4",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node8":{           "core":"solrj_collection4_shard4_replica_n5",           "base_url":"http://127.0.0.1:41412",           "node_name":"127.0.0.1:41412_",           "state":"down",           "type":"NRT"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node9/data/",           "base_url":"http://127.0.0.1:37137",           "node_name":"127.0.0.1:37137_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",           "core":"solrj_collection4_shard5_replica_n6",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"text",     "name":"compositeId"},   "maxShardsPerNode":"5",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37137_, 127.0.0.1:37317_, 127.0.0.1:41412_, 127.0.0.1:45178_]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection4//collections/solrj_collection4/state.json/29)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node3":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node3/data/",
          "base_url":"http://127.0.0.1:45178",
          "node_name":"127.0.0.1:45178_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node3/data/tlog",
          "core":"solrj_collection4_shard1_replica_n1",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node7/data/",
          "base_url":"http://127.0.0.1:37317",
          "node_name":"127.0.0.1:37317_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:39619/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",
          "core":&