[JENKINS] Lucene-Solr-Tests-master - Build # 2163 - Unstable

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
15 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2163 - Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2163/

3 tests failed.
FAILED:  org.apache.solr.cloud.AddReplicaTest.test

Error Message:
core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"https://127.0.0.1:37363/solr","node_name":"127.0.0.1:37363_solr","state":"active","type":"NRT"}

Stack Trace:
java.lang.AssertionError: core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"https://127.0.0.1:37363/solr","node_name":"127.0.0.1:37363_solr","state":"active","type":"NRT"}
        at __randomizedtesting.SeedInfo.seed([2CFC9153E32D2E19:A4A8AE894DD143E1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.AddReplicaTest.test(AddReplicaTest.java:84)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
        at __randomizedtesting.SeedInfo.seed([2CFC9153E32D2E19:7BD56AE638DFCC02]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi(AutoScalingHandlerTest.java:711)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.security.TestAuthorizationFramework.authorizationFrameworkTest

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([2CFC9153E32D2E19:625FE480F2F63F09]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 38 more




Build Log:
[...truncated 11683 lines...]
   [junit4] Suite: org.apache.solr.security.TestAuthorizationFramework
   [junit4]   2> 636045 INFO  (SUITE-TestAuthorizationFramework-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/init-core-data-001
   [junit4]   2> 636046 WARN  (SUITE-TestAuthorizationFramework-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 636046 INFO  (SUITE-TestAuthorizationFramework-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 636102 INFO  (SUITE-TestAuthorizationFramework-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 636102 INFO  (SUITE-TestAuthorizationFramework-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 636136 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 636153 INFO  (Thread-232) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 636153 INFO  (Thread-232) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 636198 ERROR (Thread-232) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 636262 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkTestServer start zk server on port:41566
   [junit4]   2> 636741 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 636755 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 636756 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 636757 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 636757 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 636758 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 636791 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 636792 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 636793 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 636793 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 636794 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 636937 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 638789 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 638870 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d166f1e{/,null,AVAILABLE}
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@42d56608{HTTP/1.1,[http/1.1]}{127.0.0.1:44207}
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server Started @651951ms
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=44207, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/control-001/cores}
   [junit4]   2> 638871 ERROR (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 638871 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-06T16:14:25.680Z
   [junit4]   2> 639060 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 639061 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/control-001/solr.xml
   [junit4]   2> 639064 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 639064 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 639151 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 639179 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41566/solr
   [junit4]   2> 639369 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f921c83060004, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 640057 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 640077 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44207_
   [junit4]   2> 640078 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.Overseer Overseer (id=98958367980388357-127.0.0.1:44207_-n_0000000000) starting
   [junit4]   2> 640167 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44207_
   [junit4]   2> 640188 INFO  (zkCallback-153-thread-1-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 640300 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.CoreContainer Initializing authorization plugin: org.apache.solr.security.MockAuthorizationPlugin
   [junit4]   2> 640576 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 640650 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 640650 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 640665 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:44207_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/control-001/cores
   [junit4]   2> 640904 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 640904 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41566/solr ready
   [junit4]   2> 641023 INFO  (qtp1973469344-1039) [n:127.0.0.1:44207_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 641023 INFO  (qtp1973469344-1039) [n:127.0.0.1:44207_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44207_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 641237 INFO  (OverseerThreadFactory-425-thread-1-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 641714 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 641714 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 641715 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 641909 INFO  (zkCallback-153-thread-1-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 643058 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 643184 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 644236 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 644417 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 644417 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 644417 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 644418 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 644543 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=94.48828125, floorSegmentMB=1.314453125, forceMergeDeletesPctAllowed=13.873212730879272, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10439239699921332
   [junit4]   2> 644568 WARN  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 645279 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 645279 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 645280 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 645280 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 645281 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5480657186537028]
   [junit4]   2> 645389 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5aff803d[control_collection_shard1_replica_n1] main]
   [junit4]   2> 645422 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 645422 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 645422 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 645423 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583333897385541632
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44207/control_collection_shard1_replica_n1/
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:44207/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 645445 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 645481 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44207/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 645481 INFO  (searcherExecutor-428-thread-1-processing-n:127.0.0.1:44207_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@5aff803d[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 645496 INFO  (zkCallback-153-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 645531 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 645533 INFO  (qtp1973469344-1037) [n:127.0.0.1:44207_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3819
   [junit4]   2> 645570 INFO  (qtp1973469344-1039) [n:127.0.0.1:44207_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 645678 INFO  (zkCallback-153-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 646576 INFO  (qtp1973469344-1039) [n:127.0.0.1:44207_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44207_&wt=javabin&version=2} status=0 QTime=5554
   [junit4]   2> 646703 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 646704 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41566/solr ready
   [junit4]   2> 646704 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 646719 INFO  (qtp1973469344-1040) [n:127.0.0.1:44207_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 646719 INFO  (qtp1973469344-1040) [n:127.0.0.1:44207_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 646756 INFO  (OverseerThreadFactory-425-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 646756 INFO  (OverseerCollectionConfigSetProcessor-98958367980388357-127.0.0.1:44207_-n_0000000000) [n:127.0.0.1:44207_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 646757 WARN  (OverseerThreadFactory-425-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 646995 INFO  (qtp1973469344-1040) [n:127.0.0.1:44207_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 646995 INFO  (qtp1973469344-1040) [n:127.0.0.1:44207_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=276
   [junit4]   2> 648020 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001 of type NRT
   [junit4]   2> 648036 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 648198 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f8fba45{/,null,AVAILABLE}
   [junit4]   2> 648198 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@449626b{HTTP/1.1,[http/1.1]}{127.0.0.1:40892}
   [junit4]   2> 648198 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server Started @661278ms
   [junit4]   2> 648198 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=40892, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001/cores}
   [junit4]   2> 648199 ERROR (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 648199 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 648199 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 648199 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 648199 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-06T16:14:35.008Z
   [junit4]   2> 648311 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 648311 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001/solr.xml
   [junit4]   2> 648367 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 648367 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 648402 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 648454 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41566/solr
   [junit4]   2> 648692 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 648693 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 648712 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40892_
   [junit4]   2> 648763 INFO  (zkCallback-153-thread-1-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 648873 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 648800 INFO  (zkCallback-161-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 648784 INFO  (OverseerCollectionConfigSetProcessor-98958367980388357-127.0.0.1:44207_-n_0000000000) [n:127.0.0.1:44207_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 649006 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.c.CoreContainer Initializing authorization plugin: org.apache.solr.security.MockAuthorizationPlugin
   [junit4]   2> 649459 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 649496 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 649496 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 649498 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:40892_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001/cores
   [junit4]   2> 649747 INFO  (qtp326318319-1086) [n:127.0.0.1:40892_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 649747 INFO  (qtp326318319-1086) [n:127.0.0.1:40892_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:40892_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 649781 INFO  (OverseerThreadFactory-425-thread-3-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:40892_ for creating new replica
   [junit4]   2> 649785 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 649785 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 649786 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 649838 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 650936 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 651032 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 651869 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 651995 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 651997 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 651997 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 651997 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
   [junit4]   2> 652068 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=94.48828125, floorSegmentMB=1.314453125, forceMergeDeletesPctAllowed=13.873212730879272, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10439239699921332
   [junit4]   2> 652088 WARN  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 652514 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 652514 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 652515 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 652515 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 652516 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5480657186537028]
   [junit4]   2> 652534 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@17284896[collection1_shard2_replica_n41] main]
   [junit4]   2> 652535 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 652535 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 652536 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 652536 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583333904844062720
   [junit4]   2> 652624 INFO  (searcherExecutor-439-thread-1-processing-n:127.0.0.1:40892_ x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@17284896[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40892/collection1_shard2_replica_n41/
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:40892/collection1_shard2_replica_n41/ has no replicas
   [junit4]   2> 652626 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 652681 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40892/collection1_shard2_replica_n41/ shard2
   [junit4]   2> 652698 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 652733 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 652735 INFO  (qtp326318319-1084) [n:127.0.0.1:40892_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2949
   [junit4]   2> 652737 INFO  (qtp326318319-1086) [n:127.0.0.1:40892_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:40892_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=2989
   [junit4]   2> 652844 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 653775 INFO  (OverseerCollectionConfigSetProcessor-98958367980388357-127.0.0.1:44207_-n_0000000000) [n:127.0.0.1:44207_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 653868 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001 of type NRT
   [junit4]   2> 653869 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 653941 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26fc0fc8{/,null,AVAILABLE}
   [junit4]   2> 653942 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4c1a0e3a{HTTP/1.1,[http/1.1]}{127.0.0.1:37009}
   [junit4]   2> 653942 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server Started @667022ms
   [junit4]   2> 653942 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=37009, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001/cores}
   [junit4]   2> 653942 ERROR (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 653943 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 653943 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 653943 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 653943 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-06T16:14:40.752Z
   [junit4]   2> 654032 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 654033 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001/solr.xml
   [junit4]   2> 654070 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 654070 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 654071 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 654161 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41566/solr
   [junit4]   2> 654254 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 654292 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 654293 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37009_
   [junit4]   2> 654303 INFO  (zkCallback-161-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 654597 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 654600 INFO  (zkCallback-153-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 654604 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 654696 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.c.CoreContainer Initializing authorization plugin: org.apache.solr.security.MockAuthorizationPlugin
   [junit4]   2> 654916 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 654954 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 654987 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 654989 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37009_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001/cores
   [junit4]   2> 655244 INFO  (qtp1175868822-1115) [n:127.0.0.1:37009_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 655244 INFO  (qtp1175868822-1115) [n:127.0.0.1:37009_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37009_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 655276 INFO  (OverseerThreadFactory-425-thread-4-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:37009_ for creating new replica
   [junit4]   2> 655279 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 655279 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 655279 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 655371 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 655385 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 656615 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 656706 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
   [junit4]   2> 657763 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 657890 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
   [junit4]   2> 657891 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 657891 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 657891 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
   [junit4]   2> 657893 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=94.48828125, floorSegmentMB=1.314453125, forceMergeDeletesPctAllowed=13.873212730879272, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10439239699921332
   [junit4]   2> 657908 WARN  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 658195 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 658195 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 658196 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 658196 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 658197 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5480657186537028]
   [junit4]   2> 658198 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@aa396cd[collection1_shard1_replica_n43] main]
   [junit4]   2> 658200 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 658200 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 658201 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 658201 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583333910784245760
   [junit4]   2> 658269 INFO  (searcherExecutor-450-thread-1-processing-n:127.0.0.1:37009_ x:collection1_shard1_replica_n43 s:shard1 c:collection1) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@aa396cd[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37009/collection1_shard1_replica_n43/
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:37009/collection1_shard1_replica_n43/ has no replicas
   [junit4]   2> 658276 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 658311 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37009/collection1_shard1_replica_n43/ shard1
   [junit4]   2> 658312 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 658325 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 658362 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 658364 INFO  (qtp1175868822-1113) [n:127.0.0.1:37009_ c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3085
   [junit4]   2> 658380 INFO  (qtp1175868822-1115) [n:127.0.0.1:37009_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:37009_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3136
   [junit4]   2> 658472 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 658472 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 659391 INFO  (OverseerCollectionConfigSetProcessor-98958367980388357-127.0.0.1:44207_-n_0000000000) [n:127.0.0.1:44207_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 659844 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001 of type NRT
   [junit4]   2> 659863 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 659900 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16f69f8a{/,null,AVAILABLE}
   [junit4]   2> 659900 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@44f9c8ca{HTTP/1.1,[http/1.1]}{127.0.0.1:37776}
   [junit4]   2> 659900 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.e.j.s.Server Started @672980ms
   [junit4]   2> 659900 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=37776, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001/cores}
   [junit4]   2> 659901 ERROR (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 659916 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 659916 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 659916 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 659916 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-06T16:14:46.725Z
   [junit4]   2> 659990 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 659990 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001/solr.xml
   [junit4]   2> 660026 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 660026 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 660027 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 660101 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41566/solr
   [junit4]   2> 660290 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 660307 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 660310 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37776_
   [junit4]   2> 660538 INFO  (zkCallback-178-thread-1-processing-n:127.0.0.1:37776_) [n:127.0.0.1:37776_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 660612 INFO  (zkCallback-161-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 660612 INFO  (zkCallback-153-thread-2-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 660541 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 660538 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 660887 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.c.CoreContainer Initializing authorization plugin: org.apache.solr.security.MockAuthorizationPlugin
   [junit4]   2> 661381 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 661529 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 661529 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 661616 INFO  (TEST-TestAuthorizationFramework.authorizationFrameworkTest-seed#[2CFC9153E32D2E19]) [n:127.0.0.1:37776_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001/cores
   [junit4]   2> 662647 INFO  (qtp1175868822-1116) [n:127.0.0.1:37009_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 662647 INFO  (qtp1175868822-1116) [n:127.0.0.1:37009_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37776_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 662661 INFO  (OverseerThreadFactory-425-thread-5-processing-n:127.0.0.1:44207_) [n:127.0.0.1:44207_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:37776_ for creating new replica
   [junit4]   2> 662988 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_    ] o.a.s.s.MockAuthorizationPlugin User request: null
   [junit4]   2> 662988 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 662988 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 663025 INFO  (zkCallback-166-thread-1-processing-n:127.0.0.1:40892_) [n:127.0.0.1:40892_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 663207 INFO  (zkCallback-172-thread-1-processing-n:127.0.0.1:37009_) [n:127.0.0.1:37009_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 663227 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 663369 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
   [junit4]   2> 664541 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 664672 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
   [junit4]   2> 664672 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@365fb823
   [junit4]   2> 664672 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 664672 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.TestAuthorizationFramework_2CFC9153E32D2E19-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
   [junit4]   2> 664674 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=94.48828125, floorSegmentMB=1.314453125, forceMergeDeletesPctAllowed=13.873212730879272, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10439239699921332
   [junit4]   2> 664691 WARN  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 665074 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 665074 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 665075 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 665075 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 665076 INFO  (qtp428924352-1144) [n:127.0.0.1:37776_ c:collection1 s:shard2  x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping clas

[...truncated too long message...]

DDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1509989623563000000],
   [junit4]   2>     "_enqueue_time_":1509989623563000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:42243_solr"]}}
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$2(ScheduledTriggers.java:251)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
   [junit4]   2>   "id":"14f48f8ff6ff98c0T42er2m8p9054ro2nh6btjrd7q",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1509989623563000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1509989623563000000],
   [junit4]   2>     "_enqueue_time_":1509989623563000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:42243_solr"]}}
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:121)
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$2(ScheduledTriggers.java:248)
   [junit4]   2> ... 6 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=node_added_trigger1/14f48f8ff6ff98c0T42er2m8p9054ro2nh6btjrd7q/0&waitForFinalState=true&collection=readApiTestViolations&targetNode=127.0.0.1:42243_solr&replica=core_node5
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:115)
   [junit4]   2> ... 7 more
   [junit4]   2> Caused by: java.lang.InterruptedException: sleep interrupted
   [junit4]   2> at java.lang.Thread.sleep(Native Method)
   [junit4]   2> at java.lang.Thread.sleep(Thread.java:340)
   [junit4]   2> at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:386)
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:161)
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:85)
   [junit4]   2> ... 7 more
   [junit4]   2> 5402501 INFO  (jetty-closer-2528-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@45579bdb{/solr,null,UNAVAILABLE}
   [junit4]   2> 5402502 ERROR (SUITE-AutoScalingHandlerTest-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 5402534 INFO  (SUITE-AutoScalingHandlerTest-seed#[2CFC9153E32D2E19]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39112 39112
   [junit4]   2> 5402668 INFO  (Thread-9401) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39112 39112
   [junit4]   2> 5402704 WARN  (Thread-9401) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 209 /solr/aliases.json
   [junit4]   2> 209 /solr/clusterprops.json
   [junit4]   2> 4 /solr/configs/conf
   [junit4]   2> 3 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 209 /solr/clusterstate.json
   [junit4]   2> 34 /solr/collections/readApiTestViolations/state.json
   [junit4]   2> 5 /solr/collections/COLL1/state.json
   [junit4]   2> 2 /solr/collections/readApiTestViolations/leader_elect/shard1/election/98958677963571206-core_node3-n_0000000001
   [junit4]   2> 2 /solr/collections/readApiTestViolations/leader_elect/shard1/election/98958677963571204-core_node9-n_0000000003
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 209 /solr/live_nodes
   [junit4]   2> 209 /solr/collections
   [junit4]   2> 2 /solr/autoscaling/events/node_lost_trigger
   [junit4]   2> 2 /solr/autoscaling/events/node_added_trigger
   [junit4]   2> 2 /solr/autoscaling/events/node_added_trigger1
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_2CFC9153E32D2E19-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=251, maxMBSortInHeap=7.960235860438525, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6e89e0), locale=be, timezone=SystemV/HST10
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=133544256,total=526909440
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecoveryHdfs, SimpleFacetsTest, SolrInfoBeanTest, ResponseLogComponentTest, TestDistribDocBasedVersion, TestSolrCloudSnapshots, ChangedSchemaMergeTest, TestSuggestSpellingConverter, TestCollapseQParserPlugin, HttpSolrCallGetCoreTest, DocValuesTest, JavabinLoaderTest, ExternalFileFieldSortTest, ClassificationUpdateProcessorIntegrationTest, CdcrReplicationDistributedZkTest, FacetPivotSmallTest, TestComponentsName, BaseCdcrDistributedZkTest, TestCloudPivotFacet, ZkStateWriterTest, TestCollationFieldDocValues, LeaderInitiatedRecoveryOnCommitTest, ReplicationFactorTest, CacheHeaderTest, TestReplicationHandler, DistributedQueryComponentCustomSortTest, TestSolrCloudWithSecureImpersonation, DistribCursorPagingTest, TermsComponentTest, TestPerFieldSimilarityWithDefaultOverride, ConnectionManagerTest, TestSegmentSorting, UpdateParamsTest, UpdateLogTest, TestRestoreCore, TestConfig, CursorMarkTest, NoCacheHeaderTest, TestRandomCollapseQParserPlugin, SpatialFilterTest, TestRequestForwarding, TestSlowCompositeReaderWrapper, SolrJmxReporterCloudTest, AtomicUpdateProcessorFactoryTest, FastVectorHighlighterTest, SecurityConfHandlerTest, HdfsRestartWhileUpdatingTest, DistanceUnitsTest, TestImpersonationWithHadoopAuth, TestXmlQParserPlugin, SpellCheckCollatorTest, FileUtilsTest, TestSolrDeletionPolicy1, MigrateRouteKeyTest, TestManagedSchemaAPI, TestSimpleTextCodec, TestSolrQueryResponse, TestMultiWordSynonyms, TestCustomSort, TestDFRSimilarityFactory, DateMathParserTest, QueryResultKeyTest, SoftAutoCommitTest, TestFoldingMultitermQuery, TestBulkSchemaConcurrent, TestHdfsBackupRestoreCore, TestRuleBasedAuthorizationPlugin, RankQueryTest, TestPullReplica, DeleteInactiveReplicaTest, TestHttpShardHandlerFactory, TestReversedWildcardFilterFactory, TestRandomRequestDistribution, TestCustomDocTransformer, TestCoreAdminApis, TestRandomFaceting, TestCloudDeleteByQuery, TestSchemaManager, UpdateRequestProcessorFactoryTest, CleanupOldIndexTest, TestLegacyNumericUtils, TestManagedResourceStorage, ShufflingReplicaListTransformerTest, TestUtils, SystemLogListenerTest, TestHdfsUpdateLog, TestSchemaSimilarityResource, CheckHdfsIndexTest, LeaderElectionContextKeyTest, TestImplicitCoreProperties, TestFieldCollectionResource, SolrCloudReportersTest, RulesTest, DateFieldTest, StressHdfsTest, MoveReplicaHDFSTest, TestLeaderInitiatedRecoveryThread, NodeAddedTriggerTest, RAMDirectoryFactoryTest, SpellCheckComponentTest, DocExpirationUpdateProcessorFactoryTest, PeerSyncTest, LeaderElectionIntegrationTest, TestJoin, TestApiFramework, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestSolrIndexConfig, TestNestedDocsSort, TestStreamBody, TestSubQueryTransformerCrossCore, MultiTermTest, V2ApiIntegrationTest, TestRecovery, PreAnalyzedUpdateProcessorTest, JSONWriterTest, BJQParserTest, PreAnalyzedFieldManagedSchemaCloudTest, CdcrUpdateLogTest, ExplicitHLLTest, TestCSVLoader, SpellCheckCollatorWithCollapseTest, DocumentBuilderTest, TestConfigSetsAPIZkFailure, CdcrVersionReplicationTest, TestPostingsSolrHighlighter, ZkControllerTest, TestStressVersions, StatsComponentTest, TestIndexSearcher, AnalysisAfterCoreReloadTest, CoreAdminHandlerTest, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, PathHierarchyTokenizerFactoryTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCSVResponseWriter, MBeansHandlerTest, SolrIndexConfigTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestBinaryField, MinimalSchemaTest, OutputWriterTest, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, ScriptEngineTest, URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeTest, DistributedIntervalFacetingTest, TestHighlightDedupGrouping, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, ActionThrottleTest, AssignTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, LegacyCloudClusterPropTest, MissingSegmentRecoveryTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, RecoveryAfterSoftCommitTest, ShardSplitTest, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, AutoScalingHandlerTest]
   [junit4] Completed [570/749 (3!)] on J0 in 36.24s, 7 tests, 1 failure <<< FAILURES!

[...truncated 47812 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2164 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2164/

7 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:40278/pn_riv/xa

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40278/pn_riv/xa
        at __randomizedtesting.SeedInfo.seed([4431FBB8A2913440:CC65C4620C6D59B8]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:183)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:200)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:409)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 44 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
36 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=13736, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13700, name=qtp141249648-13700, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13701, name=org.eclipse.jetty.server.session.HashSessionManager@c0fac35Timer, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13730, name=zkCallback-2226-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13753, name=searcherExecutor-5770-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13741, name=ScheduledTrigger-5767-thread-2, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=13733, name=OverseerStateUpdate-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=13706, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=13713, name=zkCallback-2226-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=13714, name=Thread-2685, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1723)   11) Thread[id=13765, name=zkCallback-2226-thread-8, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=13738, name=OverseerThreadFactory-5769-thread-1, state=WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=13705, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=13712, name=zkCallback-2226-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=13717, name=Scheduler-747633545, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=13739, name=ScheduledTrigger-5767-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=13697, name=qtp141249648-13697, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=13699, name=qtp141249648-13699, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=13695, name=qtp141249648-13695-acceptor-0@15d86183-ServerConnector@22a4bac2{HTTP/1.1,[http/1.1]}{127.0.0.1:47827}, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=13694, name=qtp141249648-13694, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=13711, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[4431FBB8A2913440]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   22) Thread[id=13759, name=indexFetcher-5776-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=13721, name=zkCallback-2226-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=13696, name=qtp141249648-13696, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=13763, name=AutoscalingActionExecutor-5768-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=13693, name=qtp141249648-13693, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=13710, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[4431FBB8A2913440]-SendThread(127.0.0.1:44719), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   28) Thread[id=13722, name=zkCallback-2226-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=13751, name=ScheduledTrigger-5767-thread-4, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=13764, name=zkCallback-2226-thread-7, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=13737, name=OverseerAutoScalingTriggerThread-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=WAITING, group=Overseer autoscaling triggers]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=13720, name=searcherExecutor-5761-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   33) Thread[id=13747, name=ScheduledTrigger-5767-thread-3, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=13723, name=zkCallback-2226-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=13735, name=OverseerCollectionConfigSetProcessor-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=WAITING, group=Overseer collection creation process.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)         at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)         at org.apache.solr.common.cloud.SolrZkClient$$Lambda$213/1353307948.execute(Unknown Source)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=13698, name=qtp141249648-13698, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 36 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
   1) Thread[id=13736, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13700, name=qtp141249648-13700, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13701, name=org.eclipse.jetty.server.session.HashSessionManager@c0fac35Timer, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13730, name=zkCallback-2226-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13753, name=searcherExecutor-5770-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13741, name=ScheduledTrigger-5767-thread-2, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=13733, name=OverseerStateUpdate-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=13706, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=13713, name=zkCallback-2226-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=13714, name=Thread-2685, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1723)
  11) Thread[id=13765, name=zkCallback-2226-thread-8, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=13738, name=OverseerThreadFactory-5769-thread-1, state=WAITING, group=Overseer collection creation process.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=13705, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=13712, name=zkCallback-2226-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=13717, name=Scheduler-747633545, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=13739, name=ScheduledTrigger-5767-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=13697, name=qtp141249648-13697, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=13699, name=qtp141249648-13699, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=13695, name=qtp141249648-13695-acceptor-0@15d86183-ServerConnector@22a4bac2{HTTP/1.1,[http/1.1]}{127.0.0.1:47827}, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=13694, name=qtp141249648-13694, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=13711, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[4431FBB8A2913440]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  22) Thread[id=13759, name=indexFetcher-5776-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=13721, name=zkCallback-2226-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=13696, name=qtp141249648-13696, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=13763, name=AutoscalingActionExecutor-5768-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=13693, name=qtp141249648-13693, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=13710, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[4431FBB8A2913440]-SendThread(127.0.0.1:44719), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  28) Thread[id=13722, name=zkCallback-2226-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=13751, name=ScheduledTrigger-5767-thread-4, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=13764, name=zkCallback-2226-thread-7, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  31) Thread[id=13737, name=OverseerAutoScalingTriggerThread-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=WAITING, group=Overseer autoscaling triggers]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=13720, name=searcherExecutor-5761-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  33) Thread[id=13747, name=ScheduledTrigger-5767-thread-3, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  34) Thread[id=13723, name=zkCallback-2226-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  35) Thread[id=13735, name=OverseerCollectionConfigSetProcessor-98960684385435657-127.0.0.1:47827_pn_riv%2Fxa-n_0000000001, state=WAITING, group=Overseer collection creation process.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.SolrZkClient$$Lambda$213/1353307948.execute(Unknown Source)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
  36) Thread[id=13698, name=qtp141249648-13698, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([4431FBB8A2913440]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=13701, name=org.eclipse.jetty.server.session.HashSessionManager@c0fac35Timer, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13730, name=zkCallback-2226-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13753, name=searcherExecutor-5770-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13741, name=ScheduledTrigger-5767-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13721, name=zkCallback-2226-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13713, name=zkCallback-2226-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=13696, name=qtp141249648-13696, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=13714, name=Thread-2685, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1723)    9) Thread[id=13763, name=AutoscalingActionExecutor-5768-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=13693, name=qtp141249648-13693, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=13710, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[4431FBB8A2913440]-SendThread(127.0.0.1:44719), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   12) Thread[id=13722, name=zkCallback-2226-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=13751, name=ScheduledTrigger-5767-thread-4, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=13765, name=zkCallback-2226-thread-8, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=13764, name=zkCallback-2226-thread-7, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=13720, name=searcherExecutor-5761-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=13712, name=zkCallback-2226-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=13747, name=ScheduledTrigger-5767-thread-3, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=13717, name=Scheduler-747633545, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=13723, name=zkCallback-2226-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=13739, name=ScheduledTrigger-5767-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=13759, name=indexFetcher-5776-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=13701, name=org.eclipse.jetty.server.session.HashSessionManager@c0fac35Timer, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13730, name=zkCallback-2226-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13753, name=searcherExecutor-5770-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13741, name=ScheduledTrigger-5767-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13721, name=zkCallback-2226-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13713, name=zkCallback-2226-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=13696, name=qtp141249648-13696, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
 

[...truncated too long message...]

ry=solr.collection.localShardsTestColl.shard2.leader, tag=1803231946
   [junit4]   2> 676275 INFO  (coreCloseExecutor-746-thread-5) [n:127.0.0.1:56794_solr c:overwrite s:shard1 r:core_node2 x:overwrite_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.overwrite.shard1.leader, tag=302485107
   [junit4]   2> 676275 INFO  (coreCloseExecutor-745-thread-1) [n:127.0.0.1:58995_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=1294412861
   [junit4]   2> 676276 INFO  (coreCloseExecutor-745-thread-5) [n:127.0.0.1:58995_solr c:foo s:shard1 r:core_node3 x:foo_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=654725334
   [junit4]   2> 676276 INFO  (coreCloseExecutor-745-thread-6) [n:127.0.0.1:58995_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard2.leader, tag=699839053
   [junit4]   2> 676309 INFO  (coreCloseExecutor-746-thread-4) [n:127.0.0.1:56794_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.foo.shard1.replica_n2, tag=1977441702
   [junit4]   2> 676309 INFO  (coreCloseExecutor-746-thread-4) [n:127.0.0.1:56794_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7d9c4149: rootName = solr_56794, domain = solr.core.foo.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.foo.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@2046c25a
   [junit4]   2> 676372 INFO  (coreCloseExecutor-746-thread-6) [n:127.0.0.1:56794_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection2.shard1.replica_n1, tag=331436859
   [junit4]   2> 676372 INFO  (coreCloseExecutor-746-thread-6) [n:127.0.0.1:56794_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2d7dbe97: rootName = solr_56794, domain = solr.core.multicollection2.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.multicollection2.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2b69c64b
   [junit4]   2> 676410 INFO  (coreCloseExecutor-746-thread-7) [n:127.0.0.1:56794_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.2nd_collection.shard2.replica_n2, tag=910687261
   [junit4]   2> 676410 INFO  (coreCloseExecutor-746-thread-7) [n:127.0.0.1:56794_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6e2abcd5: rootName = solr_56794, domain = solr.core.2nd_collection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.2nd_collection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@661bc113
   [junit4]   2> 676444 INFO  (coreCloseExecutor-746-thread-6) [n:127.0.0.1:56794_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection2.shard1.leader, tag=331436859
   [junit4]   2> 676445 INFO  (coreCloseExecutor-746-thread-4) [n:127.0.0.1:56794_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=1977441702
   [junit4]   2> 676445 INFO  (coreCloseExecutor-746-thread-1) [n:127.0.0.1:56794_solr c:localShardsTestColl s:shard1 r:core_node2 x:localShardsTestColl_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=534786383
   [junit4]   2> 676445 INFO  (coreCloseExecutor-746-thread-7) [n:127.0.0.1:56794_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard2.leader, tag=910687261
   [junit4]   2> 679212 INFO  (jetty-closer-144-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7eb5a60c{/solr,null,UNAVAILABLE}
   [junit4]   2> 679215 INFO  (jetty-closer-144-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98960766025007111-127.0.0.1:38887_solr-n_0000000000) closing
   [junit4]   2> 679215 INFO  (OverseerStateUpdate-98960766025007111-127.0.0.1:38887_solr-n_0000000000) [n:127.0.0.1:38887_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:38887_solr
   [junit4]   2> 681502 WARN  (OverseerAutoScalingTriggerThread-98960766025007111-127.0.0.1:38887_solr-n_0000000000) [n:127.0.0.1:38887_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 681523 INFO  (jetty-closer-144-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4a629c74{/solr,null,UNAVAILABLE}
   [junit4]   2> 681523 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f944ad99f0007, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 681524 INFO  (jetty-closer-144-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6450734b{/solr,null,UNAVAILABLE}
   [junit4]   2> 681542 ERROR (SUITE-CloudSolrClientTest-seed#[CFBBA8AE33933224]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 681542 INFO  (SUITE-CloudSolrClientTest-seed#[CFBBA8AE33933224]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51606 51606
   [junit4]   2> 681718 INFO  (Thread-202) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51606 51606
   [junit4]   2> 681721 WARN  (Thread-202) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 9 /solr/clusterprops.json
   [junit4]   2> 9 /solr/aliases.json
   [junit4]   2> 3 /solr/security.json
   [junit4]   2> 3 /solr/configs/conf
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 34 /solr/collections/localShardsTestColl/state.json
   [junit4]   2> 31 /solr/collections/foo/state.json
   [junit4]   2> 18 /solr/collections/2nd_collection/state.json
   [junit4]   2> 17 /solr/collections/collection1/state.json
   [junit4]   2> 16 /solr/collections/stale_state_test_col/state.json
   [junit4]   2> 14 /solr/collections/nemesis/state.json
   [junit4]   2> 14 /solr/collections/multicollection1/state.json
   [junit4]   2> 11 /solr/collections/multicollection2/state.json
   [junit4]   2> 9 /solr/clusterstate.json
   [junit4]   2> 6 /solr/collections/overwrite/state.json
   [junit4]   2> 2 /solr/collections/localShardsTestColl/leader_elect/shard3/election/98960766025007111-core_node18-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 9 /solr/live_nodes
   [junit4]   2> 9 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.impl.CloudSolrClientTest_CFBBA8AE33933224-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), title_s=Lucene50(blocksize=128), id=PostingsFormat(name=MockRandom), text=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Lucene70), multiDefault=DocValuesFormat(name=Direct), title_s=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1219, maxMBSortInHeap=6.227927037051457, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@70ab0bcd), locale=is, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=173144096,total=423100416
   [junit4]   2> NOTE: All tests run in this JVM: [HyperbolicSineEvaluatorTest, SolrParamTest, GreaterThanEqualToEvaluatorTest, TangentEvaluatorTest, CoalesceEvaluatorTest, LargeVolumeJettyTest, MultiplyEvaluatorTest, SineEvaluatorTest, SolrExampleXMLTest, ArrayEvaluatorTest, AddEvaluatorTest, TestBatchUpdate, TestCoreAdmin, CloudSolrClientBuilderTest, TestCollectionStateWatchers, OperationsTest, TestValidatingJsonMap, SelectWithEvaluatorsTest, TestHash, NormalizeEvaluatorTest, UuidEvaluatorTest, TestCollectionAdminRequest, PowerEvaluatorTest, JdbcDriverTest, DocumentAnalysisResponseTest, LargeVolumeBinaryJettyTest, TestEmbeddedSolrServer, TestNamedListCodec, TestJavaBinCodec, SolrExceptionTest, FacetFieldTest, SolrSchemalessExampleTest, BasicHttpSolrClientTest, CloudSolrClientMultiConstructorTest, CloudSolrClientTest]
   [junit4] Completed [88/146 (2!)] on J2 in 164.19s, 16 tests, 1 error <<< FAILURES!

[...truncated 46471 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2165 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2165/

3 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([E9E7B20B7AE9D447]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([E9E7B20B7AE9D447]:0)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testRandomStream

Error Message:
--> https://127.0.0.1:42091/solr/collection1:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection

Stack Trace:
java.io.IOException: --> https://127.0.0.1:42091/solr/collection1:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
        at __randomizedtesting.SeedInfo.seed([4042744F4DA5F29A:6D3A986AA705B176]:0)
        at org.apache.solr.client.solrj.io.stream.SolrStream.read(SolrStream.java:222)
        at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.getTuples(StreamExpressionTest.java:8683)
        at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testRandomStream(StreamExpressionTest.java:927)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14330 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/init-core-data-001
   [junit4]   2> 5130166 WARN  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 5130166 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 5130185 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 5130185 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 5130186 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001
   [junit4]   2> 5130186 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 5130200 INFO  (Thread-4753) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 5130200 INFO  (Thread-4753) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 5130201 ERROR (Thread-4753) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 5130305 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:43512
   [junit4]   2> 5130433 INFO  (jetty-launcher-2542-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 5130435 INFO  (jetty-launcher-2542-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 5130562 INFO  (jetty-launcher-2542-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@55ba27c3{/solr,null,AVAILABLE}
   [junit4]   2> 5130562 INFO  (jetty-launcher-2542-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@41947df1{SSL,[ssl, http/1.1]}{127.0.0.1:43641}
   [junit4]   2> 5130562 INFO  (jetty-launcher-2542-thread-1) [    ] o.e.j.s.Server Started @5141299ms
   [junit4]   2> 5130562 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43641}
   [junit4]   2> 5130562 ERROR (jetty-launcher-2542-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 5130563 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 5130563 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 5130563 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 5130563 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-08T03:07:49.287Z
   [junit4]   2> 5130651 INFO  (jetty-launcher-2542-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@53be9f05{/solr,null,AVAILABLE}
   [junit4]   2> 5130651 INFO  (jetty-launcher-2542-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@613fce3f{SSL,[ssl, http/1.1]}{127.0.0.1:34014}
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.e.j.s.Server Started @5141389ms
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34014}
   [junit4]   2> 5130652 ERROR (jetty-launcher-2542-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 5130652 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-08T03:07:49.376Z
   [junit4]   2> 5130693 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5130765 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5130818 INFO  (jetty-launcher-2542-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43512/solr
   [junit4]   2> 5130875 INFO  (jetty-launcher-2542-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43512/solr
   [junit4]   2> 5131041 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5131095 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34014_solr
   [junit4]   2> 5131096 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.c.Overseer Overseer (id=98966599691403270-127.0.0.1:34014_solr-n_0000000000) starting
   [junit4]   2> 5131242 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34014_solr
   [junit4]   2> 5131258 INFO  (zkCallback-2555-thread-1-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5131478 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5131480 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5131481 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43641_solr
   [junit4]   2> 5131514 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 5131517 INFO  (zkCallback-2555-thread-1-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 5131590 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34014.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5131793 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34014.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5131793 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34014.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5131794 INFO  (jetty-launcher-2542-thread-2) [n:127.0.0.1:34014_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node2/.
   [junit4]   2> 5131975 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43641.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5132012 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43641.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5132012 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43641.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5132013 INFO  (jetty-launcher-2542-thread-1) [n:127.0.0.1:43641_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node1/.
   [junit4]   2> 5132247 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 5132247 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43512/solr ready
   [junit4]   2> 5132489 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5132548 INFO  (OverseerThreadFactory-5841-thread-1-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 5132659 INFO  (OverseerStateUpdate-98966599691403270-127.0.0.1:34014_solr-n_0000000000) [n:127.0.0.1:34014_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:34014/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 5132679 INFO  (OverseerStateUpdate-98966599691403270-127.0.0.1:34014_solr-n_0000000000) [n:127.0.0.1:34014_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:43641/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 5132681 INFO  (OverseerStateUpdate-98966599691403270-127.0.0.1:34014_solr-n_0000000000) [n:127.0.0.1:34014_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:34014/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 5132715 INFO  (OverseerStateUpdate-98966599691403270-127.0.0.1:34014_solr-n_0000000000) [n:127.0.0.1:34014_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:43641/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 5133126 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 5133126 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 5133126 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 5133695 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5133696 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5133720 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 5133721 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 5133721 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 5133914 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5133915 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5134094 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 5134196 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 5134262 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema [collection1_shard2_replica_t6] Schema name=minimal
   [junit4]   2> 5134297 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5134297 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t6' using configuration from collection collection1, trusted=true
   [junit4]   2> 5134297 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43641.solr.core.collection1.shard2.replica_t6' (registry 'solr.core.collection1.shard2.replica_t6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5134298 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5134306 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema [collection1_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 5134308 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5134308 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t2' using configuration from collection collection1, trusted=true
   [junit4]   2> 5134308 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43641.solr.core.collection1.shard1.replica_t2' (registry 'solr.core.collection1.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5134313 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [[collection1_shard2_replica_t6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node1/collection1_shard2_replica_t6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node1/./collection1_shard2_replica_t6/data/]
   [junit4]   2> 5134321 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5134321 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [[collection1_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node1/collection1_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node1/./collection1_shard1_replica_t2/data/]
   [junit4]   2> 5134388 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 5134398 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 5134432 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 5134434 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5134434 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
   [junit4]   2> 5134451 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34014.solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5134451 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5134451 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [[collection1_shard1_replica_t1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node2/collection1_shard1_replica_t1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node2/./collection1_shard1_replica_t1/data/]
   [junit4]   2> 5134465 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema [collection1_shard2_replica_t4] Schema name=minimal
   [junit4]   2> 5134466 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 5134466 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t4' using configuration from collection collection1, trusted=true
   [junit4]   2> 5134467 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34014.solr.core.collection1.shard2.replica_t4' (registry 'solr.core.collection1.shard2.replica_t4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@680a0bcb
   [junit4]   2> 5134467 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 5134467 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [[collection1_shard2_replica_t4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node2/collection1_shard2_replica_t4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E9E7B20B7AE9D447-001/tempDir-001/node2/./collection1_shard2_replica_t4/data/]
   [junit4]   2> 5139419 WARN  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker-SendThread(127.0.0.1:43512)) [    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 6850ms for sessionid 0x15f99991b3c0008
   [junit4]   2> 5139435 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f99991b3c0008, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 5139554 WARN  (zkCallback-2561-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@204e0e96 name: ZooKeeperConnection Watcher:127.0.0.1:43512/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 5139591 WARN  (zkCallback-2561-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 5139655 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 5139655 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5139656 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5139656 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5139658 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e54bf4d[collection1_shard2_replica_t6] main]
   [junit4]   2> 5139720 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 5139740 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 5139765 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 5139744 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 5139767 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5139767 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5139785 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5139785 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5139785 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5139788 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@9db4985[collection1_shard1_replica_t1] main]
   [junit4]   2> 5139801 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5139801 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5139803 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@55090995[collection1_shard1_replica_t2] main]
   [junit4]   2> 5139813 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 5139813 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 5139814 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 5139814 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 5139829 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583465605159190528
   [junit4]   2> 5139831 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3786ea35[collection1_shard2_replica_t4] main]
   [junit4]   2> 5139832 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 5139832 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 5139833 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5139833 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583465605163384832
   [junit4]   2> 5139834 INFO  (searcherExecutor-5846-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@7e54bf4d[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5139834 INFO  (searcherExecutor-5848-thread-1-processing-n:127.0.0.1:34014_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@9db4985[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5139851 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 5139851 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 5139852 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 5139852 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5139852 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 5139852 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583465605183307776
   [junit4]   2> 5139853 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5139853 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583465605184356352
   [junit4]   2> 5139869 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 5139870 INFO  (searcherExecutor-5849-thread-1-processing-n:127.0.0.1:34014_solr x:collection1_shard2_replica_t4 s:shard2 c:collection1 r:core_node7) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [collection1_shard2_replica_t4] Registered new searcher Searcher@3786ea35[collection1_shard2_replica_t4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5139871 INFO  (searcherExecutor-5847-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard1_replica_t2 s:shard1 c:collection1 r:core_node5) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [collection1_shard1_replica_t2] Registered new searcher Searcher@55090995[collection1_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5139897 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 5139947 INFO  (zkCallback-2555-thread-1-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5139947 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5139947 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140054 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140054 INFO  (zkCallback-2555-thread-1-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140054 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140395 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 5140395 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 5140395 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/
   [junit4]   2> 5140395 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=https://127.0.0.1:34014/solr START replicas=[https://127.0.0.1:43641/solr/collection1_shard2_replica_t6/] nUpdates=100
   [junit4]   2> 5140435 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 5140436 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 5140436 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43641/solr/collection1_shard1_replica_t2/
   [junit4]   2> 5140436 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 url=https://127.0.0.1:43641/solr START replicas=[https://127.0.0.1:34014/solr/collection1_shard1_replica_t1/] nUpdates=100
   [junit4]   2> 5140436 INFO  (qtp1912360520-16747) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=15
   [junit4]   2> 5140470 INFO  (qtp1396023391-16750) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 5140724 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=https://127.0.0.1:34014/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 5140724 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 5140724 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 5140724 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 5140725 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController collection1_shard2_replica_t4 stopping background replication from leader
   [junit4]   2> 5140727 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/ shard2
   [junit4]   2> 5140761 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 url=https://127.0.0.1:43641/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 5140761 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 5140761 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 5140761 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 5140761 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ZkController collection1_shard1_replica_t2 stopping background replication from leader
   [junit4]   2> 5140763 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43641/solr/collection1_shard1_replica_t2/ shard1
   [junit4]   2> 5140864 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140864 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140864 INFO  (zkCallback-2555-thread-2-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5140879 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 5140880 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=7753
   [junit4]   2> 5140887 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 5140923 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ZkController collection1_shard2_replica_t6 starting background replication from leader
   [junit4]   2> 5140923 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 5140944 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 5140944 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5140944 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=7222
   [junit4]   2> 5140956 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
   [junit4]   2> 5140974 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ZkController collection1_shard1_replica_t1 starting background replication from leader
   [junit4]   2> 5140974 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 5140975 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 5140975 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 5140975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
   [junit4]   2> 5140993 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=7867
   [junit4]   2> 5141010 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=7289
   [junit4]   2> 5141011 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 5141120 INFO  (zkCallback-2552-thread-2-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5141120 INFO  (zkCallback-2552-thread-1-processing-n:127.0.0.1:43641_solr) [n:127.0.0.1:43641_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5141121 INFO  (zkCallback-2555-thread-2-processing-n:127.0.0.1:34014_solr) [n:127.0.0.1:34014_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 5141626 INFO  (OverseerCollectionConfigSetProcessor-98966599691403270-127.0.0.1:34014_solr-n_0000000000) [n:127.0.0.1:34014_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 5142012 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=9522
   [junit4]   2> 5142012 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 5142012 INFO  (SUITE-TestCloudRecovery-seed#[E9E7B20B7AE9D447]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 5142271 INFO  (TEST-TestCloudRecovery.corruptedLogTest-seed#[E9E7B20B7AE9D447]) [    ] o.a.s.SolrTestCaseJ4 ###Starting corruptedLogTest
   [junit4]   2> 5142358 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1583465607777484800&distrib.from=https://127.0.0.1:43641/solr/collection1_shard1_replica_t2/&wt=javabin&version=2}{deleteByQuery=*:* (-1583465607777484800)} 0 14
   [junit4]   2> 5142358 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/&wt=javabin&version=2}{deleteByQuery=*:* (-1583465607777484800)} 0 32
   [junit4]   2> 5142358 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1583465607756513280&distrib.from=https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/&wt=javabin&version=2}{deleteByQuery=*:* (-1583465607756513280)} 0 14
   [junit4]   2> 5142359 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1583465607756513280)} 0 53
   [junit4]   2> 5142361 INFO  (qtp1396023391-16762) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 5142375 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1583465607828865024,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 5142375 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 5142375 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 5142378 INFO  (qtp1912360520-16754) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 5142390 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 15
   [junit4]   2> 5142400 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5142488 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1583465607947354112,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 5142488 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 5142489 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 5142491 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
   [junit4]   2> 5142508 INFO  (qtp1396023391-16764) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=16
   [junit4]   2> 5142904 INFO  (qtp1912360520-16756) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5143011 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5143409 INFO  (qtp1912360520-16747) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5143527 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5143912 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5143926 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
   [junit4]   2> 5143926 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:34014/solr/collection1_shard2_replica_t4/
   [junit4]   2> 5143926 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5143927 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5143927 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5143927 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5143927 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5143927 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5143928 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@5b476993[collection1_shard2_replica_t6] main]
   [junit4]   2> 5143929 INFO  (searcherExecutor-5846-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@5b476993[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5143975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
   [junit4]   2> 5143975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:43641/solr/collection1_shard1_replica_t2/
   [junit4]   2> 5143975 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5143977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5143977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5143977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5143977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5143977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5143991 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@64fcc143[collection1_shard1_replica_t1] main]
   [junit4]   2> 5143993 INFO  (searcherExecutor-5848-thread-1-processing-n:127.0.0.1:34014_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@64fcc143[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5144071 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5144417 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5144575 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5144937 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5145077 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5145469 INFO  (qtp1912360520-16747) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5145591 INFO  (qtp1396023391-16764) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5145993 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5146094 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5146528 INFO  (qtp1912360520-16747) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5146598 INFO  (qtp1396023391-16764) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5146925 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5146925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5146925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5146925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5146925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5146925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5146926 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b411fb1[collection1_shard2_replica_t6] main]
   [junit4]   2> 5146927 INFO  (searcherExecutor-5846-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@4b411fb1[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5146975 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5146975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16da6536[collection1_shard1_replica_t1] main]
   [junit4]   2> 5146976 INFO  (searcherExecutor-5848-thread-1-processing-n:127.0.0.1:34014_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@16da6536[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5147055 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5147101 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5147560 INFO  (qtp1912360520-16756) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5147621 INFO  (qtp1396023391-16764) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5148094 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5148154 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5148611 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5148680 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5149158 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5149213 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5149683 INFO  (qtp1912360520-16748) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5149741 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5149940 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5149940 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a559ce7[collection1_shard2_replica_t6] main]
   [junit4]   2> 5149943 INFO  (searcherExecutor-5846-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@7a559ce7[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5149976 INFO  (qtp1912360520-16755) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5149976 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5149977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5149977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5149977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5149977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5149977 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@45ee3437[collection1_shard1_replica_t1] main]
   [junit4]   2> 5149978 INFO  (searcherExecutor-5848-thread-1-processing-n:127.0.0.1:34014_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@45ee3437[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5150286 INFO  (qtp1912360520-16756) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5150289 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5150807 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5150822 INFO  (qtp1396023391-16765) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5151332 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5151354 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5151835 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5151917 INFO  (qtp1396023391-16752) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5152341 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5152429 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5152850 INFO  (qtp1912360520-16757) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5152925 INFO  (qtp1396023391-16763) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 5152925 INFO  (indexFetcher-5867-thread-1) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@287f9157[collection1_shard2_replica_t6] main]
   [junit4]   2> 5152927 INFO  (searcherExecutor-5846-thread-1-processing-n:127.0.0.1:43641_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:43641_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@287f9157[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 5152931 INFO  (qtp1396023391-16749) [n:127.0.0.1:34014_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 5152975 INFO  (qtp1912360520-16758) [n:127.0.0.1:43641_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 5152975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 5152975 INFO  (indexFetcher-5869-thread-1) [n:127.0.0.1:34014_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard

[...truncated too long message...]

xecutor-663-thread-1) [n:127.0.0.1:34930_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard1.leader, tag=583285413
   [junit4]   2> 702090 WARN  (OverseerAutoScalingTriggerThread-98967101986373641-127.0.0.1:45758_solr-n_0000000000) [n:127.0.0.1:45758_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 702090 INFO  (jetty-closer-255-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7c297cfc{/solr,null,UNAVAILABLE}
   [junit4]   2> 702093 INFO  (jetty-closer-255-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@60e05986{/solr,null,UNAVAILABLE}
   [junit4]   2> 702097 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f9a0e0e590009, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 703452 WARN  (zkCallback-274-thread-5-processing-n:127.0.0.1:42091_solr) [n:127.0.0.1:42091_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 703453 INFO  (jetty-closer-255-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33a3fc12{/solr,null,UNAVAILABLE}
   [junit4]   2> 703509 WARN  (zkCallback-275-thread-7-processing-n:127.0.0.1:37516_solr) [n:127.0.0.1:37516_solr    ] o.a.s.c.ZkController Unable to register nodeLost path for 127.0.0.1:37516_solr
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/nodeLost/127.0.0.1:37516_solr
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$create$7(SolrZkClient.java:357)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:357)
   [junit4]   2> at org.apache.solr.cloud.ZkController.lambda$registerLiveNodesListener$2(ZkController.java:879)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.lambda$refreshLiveNodes$1(ZkStateReader.java:749)
   [junit4]   2> at java.util.concurrent.ConcurrentHashMap$KeySetView.forEach(ConcurrentHashMap.java:4649)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:748)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.access$1000(ZkStateReader.java:72)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader$LiveNodeWatcher.refreshAndWatch(ZkStateReader.java:1154)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader$LiveNodeWatcher.process(ZkStateReader.java:1149)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:262)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 703510 INFO  (jetty-closer-255-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7cba20f1{/solr,null,UNAVAILABLE}
   [junit4]   2> 703530 ERROR (SUITE-StreamExpressionTest-seed#[4042744F4DA5F29A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 703530 INFO  (SUITE-StreamExpressionTest-seed#[4042744F4DA5F29A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45876 45876
   [junit4]   2> 703581 INFO  (Thread-318) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45876 45876
   [junit4]   2> 703582 WARN  (Thread-318) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 81 /solr/aliases.json
   [junit4]   2> 9 /solr/configs/conf
   [junit4]   2> 4 /solr/security.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 81 /solr/clusterprops.json
   [junit4]   2> 81 /solr/clusterstate.json
   [junit4]   2> 24 /solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 24 /solr/collections/destinationCollection/state.json
   [junit4]   2> 20 /solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 17 /solr/collections/collection1_collection/state.json
   [junit4]   2> 12 /solr/collections/destination/state.json
   [junit4]   2> 12 /solr/collections/workQueue1/state.json
   [junit4]   2> 10 /solr/collections/mainCorpus1/state.json
   [junit4]   2> 10 /solr/collections/destination1/state.json
   [junit4]   2> 10 /solr/collections/workQueue/state.json
   [junit4]   2> 10 /solr/collections/mainCorpus/state.json
   [junit4]   2> 2 /solr/overseer_elect/election/98967101986373641-127.0.0.1:45758_solr-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 81 /solr/live_nodes
   [junit4]   2> 81 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_4042744F4DA5F29A-001
   [junit4]   2> Nov 08, 2017 5:20:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {expr_s=PostingsFormat(name=Asserting), join2_s=FSTOrd50, multiDefault=FSTOrd50, subject=FSTOrd50, a1_s=FSTOrd50, a2_s=BlockTreeOrds(blocksize=128), test_t=BlockTreeOrds(blocksize=128), level1_s=PostingsFormat(name=Asserting), ident_s=FSTOrd50, body_t=BlockTreeOrds(blocksize=128), s_multi=BlockTreeOrds(blocksize=128), col_s=FSTOrd50, a_ss=PostingsFormat(name=LuceneVarGapFixedInterval), level2_s=PostingsFormat(name=LuceneVarGapFixedInterval), a_s=BlockTreeOrds(blocksize=128), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), side_s=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Asserting), text=BlockTreeOrds(blocksize=128)}, docValues:{expr_s=DocValuesFormat(name=Asserting), join2_s=DocValuesFormat(name=Direct), multiDefault=DocValuesFormat(name=Direct), a1_s=DocValuesFormat(name=Direct), price_f=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Lucene70), a_f=DocValuesFormat(name=Direct), a2_s=DocValuesFormat(name=Memory), join1_i=DocValuesFormat(name=Asserting), a_i=DocValuesFormat(name=Asserting), test_i=DocValuesFormat(name=Direct), join3_i=DocValuesFormat(name=Memory), field_i=DocValuesFormat(name=Direct), level1_s=DocValuesFormat(name=Asserting), s_multi=DocValuesFormat(name=Memory), col_s=DocValuesFormat(name=Direct), a_ss=DocValuesFormat(name=Lucene70), level2_s=DocValuesFormat(name=Lucene70), a_s=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Lucene70), i_multi=DocValuesFormat(name=Asserting), b_ls=DocValuesFormat(name=Direct), test_dt=DocValuesFormat(name=Asserting), ident_s=DocValuesFormat(name=Direct), order_i=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene70), side_s=DocValuesFormat(name=Memory), miles_i=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1633, maxMBSortInHeap=6.956198478355017, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2522e20e), locale=ca, timezone=Asia/Aqtobe
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=283929392,total=521666560
   [junit4]   2> NOTE: All tests run in this JVM: [FacetFieldTest, SolrExampleStreamingBinaryTest, SubtractEvaluatorTest, TestSolrProperties, SolrQueryTest, TestV2Request, HyperbolicCosineEvaluatorTest, JDBCStreamTest, NormalDistributionEvaluatorTest, ModuloEvaluatorTest, AppendEvaluatorTest, ClientUtilsTest, CorrelationEvaluatorTest, JsonValidatorTest, OrEvaluatorTest, CloudSolrClientMultiConstructorTest, AbsoluteValueEvaluatorTest, RoundEvaluatorTest, SolrExampleEmbeddedTest, HttpSolrClientSSLAuthConPoolTest, CollectionAdminRequestRequiredParamsTest, MergeIndexesEmbeddedTest, TestRetryUtil, SolrExampleBinaryTest, SolrExampleXMLTest, SolrExampleJettyTest, LargeVolumeBinaryJettyTest, JettyWebappTest, TestSpellCheckResponse, TestNamedListCodec, TestJavaBinCodec, TestDocumentObjectBinder, ContentStreamTest, ModifiableSolrParamsTest, TestFastInputStream, TestXMLEscaping, SolrDocumentTest, TestUpdateRequestCodec, FieldAnalysisResponseTest, AnlysisResponseBaseTest, TestLBHttpSolrClient, ConcurrentUpdateSolrClientTest, StreamExpressionTest]
   [junit4] Completed [146/146 (1!)] on J1 in 301.28s, 130 tests, 1 error, 9 skipped <<< FAILURES!

[...truncated 46271 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2166 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2166/

14 tests failed.
FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Unexpected exception type, expected SolrException but got org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]

Stack Trace:
junit.framework.AssertionFailedError: Unexpected exception type, expected SolrException but got org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2660)
        at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:119)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.AliasIntegrationTest.lambda$test$0(AliasIntegrationTest.java:122)
        at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2655)
        ... 40 more
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56893/solr/testalias: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testalias/select. Reason:
<pre>    Can not find: /solr/testalias/select</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        ... 48 more


FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
Error from server at https://127.0.0.1:48559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000012

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:48559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000012
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:85)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
KeeperErrorCode = Session expired for /clusterstate.json

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:5048CD86654A298C]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:308)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)
        at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:428)
        at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:442)
        at org.apache.solr.cloud.CustomCollectionTest.ensureClusterEmpty(CustomCollectionTest.java:53)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistribCursorPagingTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1198)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:840)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:839)
        at org.apache.solr.cloud.DistribCursorPagingTest.doRandomSortsOnLargeIndex(DistribCursorPagingTest.java:581)
        at org.apache.solr.cloud.DistribCursorPagingTest.test(DistribCursorPagingTest.java:93)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 50 more


FAILED:  org.apache.solr.cloud.ShardRoutingCustomTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.ShardRoutingCustomTest.doCustomSharding(ShardRoutingCustomTest.java:82)
        at org.apache.solr.cloud.ShardRoutingCustomTest.test(ShardRoutingCustomTest.java:46)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 46 more


FAILED:  org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess

Error Message:
No registered leader was found after waiting for 4000ms , collection: testPeersyncFailureReplicationSuccess slice: shard1 saw state=DocCollection(testPeersyncFailureReplicationSuccess//collections/testPeersyncFailureReplicationSuccess/state.json/10)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n1",           "base_url":"https://127.0.0.1:56855/solr",           "node_name":"127.0.0.1:56855_solr",           "state":"down",           "type":"NRT"},         "core_node4":{           "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n3",           "base_url":"https://127.0.0.1:34941/solr",           "node_name":"127.0.0.1:34941_solr",           "state":"active",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"} with live_nodes=[127.0.0.1:34941_solr]

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: testPeersyncFailureReplicationSuccess slice: shard1 saw state=DocCollection(testPeersyncFailureReplicationSuccess//collections/testPeersyncFailureReplicationSuccess/state.json/10)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n1",
          "base_url":"https://127.0.0.1:56855/solr",
          "node_name":"127.0.0.1:56855_solr",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n3",
          "base_url":"https://127.0.0.1:34941/solr",
          "node_name":"127.0.0.1:34941_solr",
          "state":"active",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:34941_solr]
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:EEF820A2896EA897]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:816)
        at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:800)
        at org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess(TestCloudSearcherWarming.java:196)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:4C975C9E19E732DB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.createCollection(TestCollectionsAPIViaSolrCloudCluster.java:91)
        at org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll(TestCollectionsAPIViaSolrCloudCluster.java:214)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressInPlaceUpdates.stressTest

Error Message:
Captured an uncaught exception in thread: Thread[id=14966, name=WRITER7, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=14966, name=WRITER7, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49233/collection1: Cannot talk to ZooKeeper - Updates are disabled.
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4]:0)
        at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:306)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49233/collection1: Cannot talk to ZooKeeper - Updates are disabled.
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.TestStressInPlaceUpdates.addDocAndGetVersion(TestStressInPlaceUpdates.java:549)
        at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:271)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSearchRate

Error Message:
{srt=[TestEvent{timestamp=1510234372953000000, config={   "trigger":"search_rate_trigger",   "afterAction":[],   "stage":[     "FAILED",     "SUCCEEDED"],   "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",   "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={   "id":"14f56e28773e7ac0Tc60vaqu1q49az5pjp22hxzsfu",   "source":"search_rate_trigger",   "eventTime":1510234370131000000,   "eventType":"SEARCHRATE",   "properties":{     "node":{       "127.0.0.1:58947_solr":8.4,       "127.0.0.1:33902_solr":7.4},     "replica":[       "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":8.4}}",       "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":7.4,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],     "collection":{"collection1":15.8},     "shard":{"collection1":{"shard1":15.8}},     "_enqueue_time_":1510234372922000000}}, message='null'}, TestEvent{timestamp=1510234378198000000, config={   "trigger":"search_rate_trigger",   "afterAction":[],   "stage":[     "FAILED",     "SUCCEEDED"],   "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",   "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={   "id":"14f56e291d5cb980Tc60vaqu1q49az5pjp22hxzsfw",   "source":"search_rate_trigger",   "eventTime":1510234372918000000,   "eventType":"SEARCHRATE",   "properties":{     "node":{       "127.0.0.1:58947_solr":10.047085058635941,       "127.0.0.1:33902_solr":8.711271600079098},     "replica":[       "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":10.047085058635941}}",       "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":8.711271600079098,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],     "collection":{"collection1":18.758356658715037},     "shard":{"collection1":{"shard1":18.758356658715037}},     "_enqueue_time_":1510234378178000000}}, message='null'}]} expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: {srt=[TestEvent{timestamp=1510234372953000000, config={
  "trigger":"search_rate_trigger",
  "afterAction":[],
  "stage":[
    "FAILED",
    "SUCCEEDED"],
  "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",
  "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={
  "id":"14f56e28773e7ac0Tc60vaqu1q49az5pjp22hxzsfu",
  "source":"search_rate_trigger",
  "eventTime":1510234370131000000,
  "eventType":"SEARCHRATE",
  "properties":{
    "node":{
      "127.0.0.1:58947_solr":8.4,
      "127.0.0.1:33902_solr":7.4},
    "replica":[
      "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":8.4}}",
      "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":7.4,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],
    "collection":{"collection1":15.8},
    "shard":{"collection1":{"shard1":15.8}},
    "_enqueue_time_":1510234372922000000}}, message='null'}, TestEvent{timestamp=1510234378198000000, config={
  "trigger":"search_rate_trigger",
  "afterAction":[],
  "stage":[
    "FAILED",
    "SUCCEEDED"],
  "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",
  "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={
  "id":"14f56e291d5cb980Tc60vaqu1q49az5pjp22hxzsfw",
  "source":"search_rate_trigger",
  "eventTime":1510234372918000000,
  "eventType":"SEARCHRATE",
  "properties":{
    "node":{
      "127.0.0.1:58947_solr":10.047085058635941,
      "127.0.0.1:33902_solr":8.711271600079098},
    "replica":[
      "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":10.047085058635941}}",
      "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":8.711271600079098,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],
    "collection":{"collection1":18.758356658715037},
    "shard":{"collection1":{"shard1":18.758356658715037}},
    "_enqueue_time_":1510234378178000000}}, message='null'}]} expected:<1> but was:<2>
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:67E15D64971639BB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSearchRate(TriggerIntegrationTest.java:1285)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:34417_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:34417_solr wasn't created
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:2213CBE156E5521B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:919)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 38 more


FAILED:  junit.framework.TestSuite.org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest

Error Message:
Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 31,113,920 bytes (threshold is 10,485,760). Field reference sizes (counted individually):   - 31,113,280 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer   - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules   - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp   - 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION   - 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels   - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName   - 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 31,113,920 bytes (threshold is 10,485,760). Field reference sizes (counted individually):
  - 31,113,280 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer
  - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules
  - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp
  - 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION
  - 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
  - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName
  - 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4]:0)
        at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:170)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:49042/_

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:49042/_
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:315)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
        at sun.security.ssl.InputRecord.read(InputRecord.java:503)
        at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
        at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
        at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 43 more


FAILED:  junit.framework.TestSuite.org.apache.solr.search.mlt.CloudMLTQParserTest

Error Message:
Could not load collection from ZK: mlt-collection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: mlt-collection
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.common.cloud.ZkStateReader.registerCollectionStateWatcher(ZkStateReader.java:1290)
        at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1320)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:440)
        at org.apache.solr.search.mlt.CloudMLTQParserTest.setupCluster(CloudMLTQParserTest.java:50)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/mlt-collection/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 30 more




Build Log:
[...truncated 11685 lines...]
   [junit4] Suite: org.apache.solr.cloud.AliasIntegrationTest
   [junit4]   2> 101553 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.AliasIntegrationTest_3AA943ED58D09FF4-001/init-core-data-001
   [junit4]   2> 101554 WARN  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=13 numCloses=13
   [junit4]   2> 101574 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 101585 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 101670 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.AliasIntegrationTest_3AA943ED58D09FF4-001/tempDir-001
   [junit4]   2> 101719 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 103046 INFO  (Thread-4) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 103354 INFO  (Thread-4) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 103663 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:50043
   [junit4]   2> 106115 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 107074 ERROR (Thread-4) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 108262 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa0ee6d490000, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 109328 INFO  (jetty-launcher-7-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 109366 INFO  (jetty-launcher-7-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 109601 INFO  (jetty-launcher-7-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4abd2d4e{/solr,null,AVAILABLE}
   [junit4]   2> 110086 INFO  (jetty-launcher-7-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@8421441{/solr,null,AVAILABLE}
   [junit4]   2> 110094 INFO  (jetty-launcher-7-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1c7af183{HTTP/1.1,[http/1.1]}{127.0.0.1:56893}
   [junit4]   2> 110094 INFO  (jetty-launcher-7-thread-1) [    ] o.e.j.s.Server Started @131189ms
   [junit4]   2> 110094 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56893}
   [junit4]   2> 110097 INFO  (jetty-launcher-7-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@67b9b98a{HTTP/1.1,[http/1.1]}{127.0.0.1:51328}
   [junit4]   2> 110101 INFO  (jetty-launcher-7-thread-2) [    ] o.e.j.s.Server Started @131196ms
   [junit4]   2> 110102 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51328}
   [junit4]   2> 110233 ERROR (jetty-launcher-7-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 110234 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 110234 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 110234 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 110234 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-09T13:18:27.147Z
   [junit4]   2> 110265 ERROR (jetty-launcher-7-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 110295 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 110296 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 110296 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 110296 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-09T13:18:27.209Z
   [junit4]   2> 110382 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 110486 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 110634 INFO  (jetty-launcher-7-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50043/solr
   [junit4]   2> 110675 INFO  (jetty-launcher-7-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50043/solr
   [junit4]   2> 112077 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa0ee6d490003, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 113564 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 113582 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51328_solr
   [junit4]   2> 113604 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.c.Overseer Overseer (id=98974662721601542-127.0.0.1:51328_solr-n_0000000000) starting
   [junit4]   2> 115073 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 115177 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56893_solr
   [junit4]   2> 115435 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51328_solr) [n:127.0.0.1:51328_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 115558 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:56893_solr) [n:127.0.0.1:56893_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 118090 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51328_solr
   [junit4]   2> 118198 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:56893_solr) [n:127.0.0.1:56893_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 118199 INFO  (OverseerStateUpdate-98974662721601542-127.0.0.1:51328_solr-n_0000000000) [n:127.0.0.1:51328_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 124295 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51328.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124296 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51328.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124296 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51328.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124314 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56893.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124314 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56893.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124314 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56893.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c768512
   [junit4]   2> 124633 INFO  (jetty-launcher-7-thread-2) [n:127.0.0.1:51328_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.AliasIntegrationTest_3AA943ED58D09FF4-001/tempDir-001/node2/.
   [junit4]   2> 124741 INFO  (jetty-launcher-7-thread-1) [n:127.0.0.1:56893_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.AliasIntegrationTest_3AA943ED58D09FF4-001/tempDir-001/node1/.
   [junit4]   2> 133887 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 133888 INFO  (SUITE-AliasIntegrationTest-seed#[3AA943ED58D09FF4]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50043/solr ready
   [junit4]   2> 133943 INFO  (TEST-AliasIntegrationTest.testErrorChecks-seed#[3AA943ED58D09FF4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testErrorChecks
   [junit4]   2> 137826 INFO  (qtp369041385-65) [n:127.0.0.1:51328_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=testErrorChecks-collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 138234 INFO  (OverseerThreadFactory-49-thread-1-processing-n:127.0.0.1:51328_solr) [n:127.0.0.1:51328_solr    ] o.a.s.c.CreateCollectionCmd Create collection testErrorChecks-collection
   [junit4]   2> 138753 INFO  (OverseerStateUpdate-98974662721601542-127.0.0.1:51328_solr-n_0000000000) [n:127.0.0.1:51328_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testErrorChecks-collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testErrorChecks-collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56893/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 138795 INFO  (OverseerStateUpdate-98974662721601542-127.0.0.1:51328_solr-n_0000000000) [n:127.0.0.1:51328_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testErrorChecks-collection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testErrorChecks-collection_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51328/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 139878 INFO  (qtp503628794-73) [n:127.0.0.1:56893_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard1_replica_n1&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 139886 INFO  (qtp503628794-73) [n:127.0.0.1:56893_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 139907 INFO  (qtp369041385-63) [n:127.0.0.1:51328_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testErrorChecks-collection_shard2_replica_n2&action=CREATE&numShards=2&collection=testErrorChecks-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 139934 INFO  (qtp369041385-63) [n:127.0.0.1:51328_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 140175 INFO  (zkCallback-18-thread-1-processing-n:127.0.0.1:56893_solr) [n:127.0.0.1:56893_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 140177 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51328_solr) [n:127.0.0.1:51328_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 140285 INFO  (zkCallback-19-thread-2-processing-n:127.0.0.1:51328_solr) [n:127.0.0.1:51328_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 140286 INFO  (zkCallback-18-thread-2-processing-n:127.0.0.1:56893_solr) [n:127.0.0.1:56893_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 141172 INFO  (qtp369041385-63) [n:127.0.0.1:51328_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 141180 INFO  (qtp503628794-73) [n:127.0.0.1:56893_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 141249 INFO  (qtp503628794-73) [n:127.0.0.1:56893_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_re

[...truncated too long message...]

ace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-core-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-core-api-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-core-avl-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-core-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-i18n-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/temp/junit4-J2-20171109_131615_5655998595048286842838.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/temp/junit4-J2-20171109_131615_5651937197852482521017.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 137.
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/build.xml:835: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/build.xml:779: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/common-build.xml:1512: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/common-build.xml:1038: At least one slave process threw an exception, first: Forked process returned with error code: 137.

Total time: 356 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-master #2165 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

Re: [JENKINS] Lucene-Solr-Tests-master - Build # 2166 - Failure

david.w.smiley@gmail.com
At least with respect to the AliasIntegrationTest failures, I believe I have an inkling to the problem and am contemplating how to fix.  It appears there is a problem with aliases similar to that of collections in terms of eventual consistency of what we see in ZooKeeper.  For collections it may have been solved (partially|completely?) with the _stateVer_ & HTTP 510 status verification.  But there is no analog of that for aliases.  


On Thu, Nov 9, 2017 at 11:24 AM Apache Jenkins Server <[hidden email]> wrote:
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2166/

14 tests failed.
FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Unexpected exception type, expected SolrException but got org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]

Stack Trace:
junit.framework.AssertionFailedError: Unexpected exception type, expected SolrException but got org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2660)
        at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:119)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51328/solr/testalias, http://127.0.0.1:56893/solr/testalias]
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at org.apache.solr.cloud.AliasIntegrationTest.lambda$test$0(AliasIntegrationTest.java:122)
        at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2655)
        ... 40 more
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56893/solr/testalias: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testalias/select. Reason:
<pre>    Can not find: /solr/testalias/select</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        ... 48 more


FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
Error from server at https://127.0.0.1:48559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000012

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:48559/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000012
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:85)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
KeeperErrorCode = Session expired for /clusterstate.json

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:5048CD86654A298C]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:308)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)
        at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:428)
        at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:442)
        at org.apache.solr.cloud.CustomCollectionTest.ensureClusterEmpty(CustomCollectionTest.java:53)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistribCursorPagingTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1198)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:840)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:839)
        at org.apache.solr.cloud.DistribCursorPagingTest.doRandomSortsOnLargeIndex(DistribCursorPagingTest.java:581)
        at org.apache.solr.cloud.DistribCursorPagingTest.test(DistribCursorPagingTest.java:93)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 50 more


FAILED:  org.apache.solr.cloud.ShardRoutingCustomTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.ShardRoutingCustomTest.doCustomSharding(ShardRoutingCustomTest.java:82)
        at org.apache.solr.cloud.ShardRoutingCustomTest.test(ShardRoutingCustomTest.java:46)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 46 more


FAILED:  org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess

Error Message:
No registered leader was found after waiting for 4000ms , collection: testPeersyncFailureReplicationSuccess slice: shard1 saw state=DocCollection(testPeersyncFailureReplicationSuccess//collections/testPeersyncFailureReplicationSuccess/state.json/10)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n1",           "base_url":"https://127.0.0.1:56855/solr",           "node_name":"127.0.0.1:56855_solr",           "state":"down",           "type":"NRT"},         "core_node4":{           "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n3",           "base_url":"https://127.0.0.1:34941/solr",           "node_name":"127.0.0.1:34941_solr",           "state":"active",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"} with live_nodes=[127.0.0.1:34941_solr]

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: testPeersyncFailureReplicationSuccess slice: shard1 saw state=DocCollection(testPeersyncFailureReplicationSuccess//collections/testPeersyncFailureReplicationSuccess/state.json/10)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n1",
          "base_url":"https://127.0.0.1:56855/solr",
          "node_name":"127.0.0.1:56855_solr",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"testPeersyncFailureReplicationSuccess_shard1_replica_n3",
          "base_url":"https://127.0.0.1:34941/solr",
          "node_name":"127.0.0.1:34941_solr",
          "state":"active",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:34941_solr]
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:EEF820A2896EA897]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:816)
        at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:800)
        at org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess(TestCloudSearcherWarming.java:196)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:4C975C9E19E732DB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.createCollection(TestCollectionsAPIViaSolrCloudCluster.java:91)
        at org.apache.solr.cloud.TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll(TestCollectionsAPIViaSolrCloudCluster.java:214)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressInPlaceUpdates.stressTest

Error Message:
Captured an uncaught exception in thread: Thread[id=14966, name=WRITER7, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=14966, name=WRITER7, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49233/collection1: Cannot talk to ZooKeeper - Updates are disabled.
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4]:0)
        at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:306)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49233/collection1: Cannot talk to ZooKeeper - Updates are disabled.
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.TestStressInPlaceUpdates.addDocAndGetVersion(TestStressInPlaceUpdates.java:549)
        at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:271)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSearchRate

Error Message:
{srt=[TestEvent{timestamp=1510234372953000000, config={   "trigger":"search_rate_trigger",   "afterAction":[],   "stage":[     "FAILED",     "SUCCEEDED"],   "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",   "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={   "id":"14f56e28773e7ac0Tc60vaqu1q49az5pjp22hxzsfu",   "source":"search_rate_trigger",   "eventTime":1510234370131000000,   "eventType":"SEARCHRATE",   "properties":{     "node":{       "127.0.0.1:58947_solr":8.4,       "127.0.0.1:33902_solr":7.4},     "replica":[       "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":8.4}}",       "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":7.4,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],     "collection":{"collection1":15.8},     "shard":{"collection1":{"shard1":15.8}},     "_enqueue_time_":1510234372922000000}}, message='null'}, TestEvent{timestamp=1510234378198000000, config={   "trigger":"search_rate_trigger",   "afterAction":[],   "stage":[     "FAILED",     "SUCCEEDED"],   "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",   "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={   "id":"14f56e291d5cb980Tc60vaqu1q49az5pjp22hxzsfw",   "source":"search_rate_trigger",   "eventTime":1510234372918000000,   "eventType":"SEARCHRATE",   "properties":{     "node":{       "127.0.0.1:58947_solr":10.047085058635941,       "127.0.0.1:33902_solr":8.711271600079098},     "replica":[       "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":10.047085058635941}}",       "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":8.711271600079098,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],     "collection":{"collection1":18.758356658715037},     "shard":{"collection1":{"shard1":18.758356658715037}},     "_enqueue_time_":1510234378178000000}}, message='null'}]} expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: {srt=[TestEvent{timestamp=1510234372953000000, config={
  "trigger":"search_rate_trigger",
  "afterAction":[],
  "stage":[
    "FAILED",
    "SUCCEEDED"],
  "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",
  "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={
  "id":"14f56e28773e7ac0Tc60vaqu1q49az5pjp22hxzsfu",
  "source":"search_rate_trigger",
  "eventTime":1510234370131000000,
  "eventType":"SEARCHRATE",
  "properties":{
    "node":{
      "127.0.0.1:58947_solr":8.4,
      "127.0.0.1:33902_solr":7.4},
    "replica":[
      "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":8.4}}",
      "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":7.4,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],
    "collection":{"collection1":15.8},
    "shard":{"collection1":{"shard1":15.8}},
    "_enqueue_time_":1510234372922000000}}, message='null'}, TestEvent{timestamp=1510234378198000000, config={
  "trigger":"search_rate_trigger",
  "afterAction":[],
  "stage":[
    "FAILED",
    "SUCCEEDED"],
  "class":"org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerListener",
  "beforeAction":[]}, stage=SUCCEEDED, actionName='null', event={
  "id":"14f56e291d5cb980Tc60vaqu1q49az5pjp22hxzsfw",
  "source":"search_rate_trigger",
  "eventTime":1510234372918000000,
  "eventType":"SEARCHRATE",
  "properties":{
    "node":{
      "127.0.0.1:58947_solr":10.047085058635941,
      "127.0.0.1:33902_solr":8.711271600079098},
    "replica":[
      "{\"core_node3\":{\n    \"base_url\":\"http://127.0.0.1:58947/solr\",\n    \"node_name\":\"127.0.0.1:58947_solr\",\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"rate\":10.047085058635941}}",
      "{\"core_node4\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"leader\":\"true\",\n    \"rate\":8.711271600079098,\n    \"base_url\":\"http://127.0.0.1:33902/solr\",\n    \"node_name\":\"127.0.0.1:33902_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\"}}"],
    "collection":{"collection1":18.758356658715037},
    "shard":{"collection1":{"shard1":18.758356658715037}},
    "_enqueue_time_":1510234378178000000}}, message='null'}]} expected:<1> but was:<2>
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:67E15D64971639BB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSearchRate(TriggerIntegrationTest.java:1285)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:34417_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:34417_solr wasn't created
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:2213CBE156E5521B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:919)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4:B2FD7C37F62CF20C]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 38 more


FAILED:  junit.framework.TestSuite.org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest

Error Message:
Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 31,113,920 bytes (threshold is 10,485,760). Field reference sizes (counted individually):   - 31,113,280 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer   - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules   - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp   - 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION   - 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels   - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName   - 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 31,113,920 bytes (threshold is 10,485,760). Field reference sizes (counted individually):
  - 31,113,280 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer
  - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules
  - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp
  - 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION
  - 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
  - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName
  - 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel
        at __randomizedtesting.SeedInfo.seed([3AA943ED58D09FF4]:0)
        at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:170)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run
--
Lucene/Solr Search Committer, Consultant, Developer, Author, Speaker
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2167 - Still unstable

Apache Jenkins Server-2
In reply to this post by Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2167/

3 tests failed.
FAILED:  org.apache.lucene.search.TestSimilarity.testSimilarity

Error Message:
expected:<2.0> but was:<1.0>

Stack Trace:
java.lang.AssertionError: expected:<2.0> but was:<1.0>
        at __randomizedtesting.SeedInfo.seed([CC0F988A905E3647:3DA0D8B290BEA35C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:443)
        at org.junit.Assert.assertEquals(Assert.java:512)
        at org.apache.lucene.search.TestSimilarity$4.collect(TestSimilarity.java:146)
        at org.apache.lucene.search.AssertingLeafCollector.collect(AssertingLeafCollector.java:52)
        at org.apache.lucene.search.AssertingCollector$1.collect(AssertingCollector.java:56)
        at org.apache.lucene.search.AssertingLeafCollector.collect(AssertingLeafCollector.java:52)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:293)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:236)
        at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:69)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:645)
        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:72)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:449)
        at org.apache.lucene.search.TestSimilarity.testSimilarity(TestSimilarity.java:137)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.search.spans.TestSpans.testSpanScorerZeroSloppyFreq

Error Message:
first doc score should be zero, 3.0794418

Stack Trace:
java.lang.AssertionError: first doc score should be zero, 3.0794418
        at __randomizedtesting.SeedInfo.seed([CC0F988A905E3647:F07C2CA126BC0340]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.lucene.search.spans.TestSpans.testSpanScorerZeroSloppyFreq(TestSpans.java:320)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:37340/forceleader_test_collection

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37340/forceleader_test_collection
        at __randomizedtesting.SeedInfo.seed([228BDCE02D54C240:C41CE82014D63B21]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
        at org.apache.solr.cloud.HttpPartitionTest.realTimeGetDocId(HttpPartitionTest.java:617)
        at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:602)
        at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:559)
        at org.apache.solr.cloud.ForceLeaderTest.bringBackOldLeaderAndSendDoc(ForceLeaderTest.java:392)
        at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:152)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 49 more




Build Log:
[...truncated 1122 lines...]
   [junit4] Suite: org.apache.lucene.search.spans.TestSpans
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSpans -Dtests.method=testSpanScorerZeroSloppyFreq -Dtests.seed=CC0F988A905E3647 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sl -Dtests.timezone=Kwajalein -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.11s J2 | TestSpans.testSpanScorerZeroSloppyFreq <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: first doc score should be zero, 3.0794418
   [junit4]    > at __randomizedtesting.SeedInfo.seed([CC0F988A905E3647:F07C2CA126BC0340]:0)
   [junit4]    > at org.apache.lucene.search.spans.TestSpans.testSpanScorerZeroSloppyFreq(TestSpans.java:320)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=FST50, id=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=Direct)}, docValues:{}, maxPointsInLeafNode=1699, maxMBSortInHeap=7.228716554511077, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@406ee11f), locale=sl, timezone=Kwajalein
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=118459952,total=357564416
   [junit4]   2> NOTE: All tests run in this JVM: [TestPayloads, TestCachingTokenFilter, TestMutablePointsReaderUtils, TestTopDocsCollector, TestDemoParallelLeafReader, TestSimpleExplanationsOfNonMatches, TestSimpleExplanationsWithFillerDocs, TestSearchForDuplicates, TestConjunctionDISI, TestRegexpQuery, TestDirectoryReader, TestSpanBoostQuery, TestSearchWithThreads, TestMultiLevelSkipList, TestVersion, TestDocValuesRewriteMethod, TestShardSearching, TestBytesRefHash, TestConjunctions, TestOneMergeWrappingMergePolicy, TestBufferedIndexInput, TestSetOnce, TestRollingUpdates, TestStringHelper, TestSameTokenSamePosition, TestDateSort, TestFixedBitSet, TestArrayUtil, TestByteBlockPool, TestTryDelete, Test2BBinaryDocValues, Test2BNumericDocValues, TestLongRangeFieldQueries, TestIndexManyDocuments, TestPackedInts, TestRollingBuffer, TestIOUtils, TestAssertions, TestIndexWriterReader, TestSloppyMath, TestFieldCacheRewriteMethod, TestBinaryDocument, TestFieldsReader, TestSegmentTermEnum, TestSimilarity2, TestSpanSearchEquivalence, TestCharsRef, TestMultiTermsEnum, TestRamUsageEstimator, TestSegmentTermDocs, TestFilterDirectory, TestField, TestMultiset, TestPerFieldPostingsFormat2, TestBlockPostingsFormat, TestPointQueries, TestLSBRadixSorter, TestSimilarityBase, TestDirectMonotonic, TestTimSorterWorstCase, TestTermScorer, TestStopFilter, TestBagOfPostings, TestReadOnlyIndex, TestFixedLengthBytesRefArray, TestFilterWeight, TestSortedSetDocValues, TestIndexWriterDeleteByQuery, TestBasics, TestIndependenceChiSquared, TestLogMergePolicy, TestMultiMMap, TestNRTThreads, Test2BBKDPoints, TestDocValues, TestLucene70NormsFormat, TestAxiomaticF3EXP, TestAutomaton, TestAttributeSource, TestPerFieldDocValuesFormat, TestIndexWriterMerging, TestFSTs, TestSloppyPhraseQuery, TestDuelingCodecs, TestStandardAnalyzer, TestNRTReaderWithThreads, TestIndexWriterForceMerge, TestByteSlices, TestConsistentFieldNumbers, TestSimpleExplanations, TestSegmentMerger, TestCollectionUtil, TestParallelCompositeReader, TestMultiThreadTermVectors, Test2BDocs, TestIndexWriterOnDiskFull, TestSpans]
   [junit4] Completed [260/475 (1!)] on J2 in 2.42s, 26 tests, 1 failure <<< FAILURES!

[...truncated 174 lines...]
   [junit4] Suite: org.apache.lucene.search.TestSimilarity
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSimilarity -Dtests.method=testSimilarity -Dtests.seed=CC0F988A905E3647 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Indianapolis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.15s J1 | TestSimilarity.testSimilarity <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<2.0> but was:<1.0>
   [junit4]    > at __randomizedtesting.SeedInfo.seed([CC0F988A905E3647:3DA0D8B290BEA35C]:0)
   [junit4]    > at org.apache.lucene.search.TestSimilarity$4.collect(TestSimilarity.java:146)
   [junit4]    > at org.apache.lucene.search.AssertingLeafCollector.collect(AssertingLeafCollector.java:52)
   [junit4]    > at org.apache.lucene.search.AssertingCollector$1.collect(AssertingCollector.java:56)
   [junit4]    > at org.apache.lucene.search.AssertingLeafCollector.collect(AssertingLeafCollector.java:52)
   [junit4]    > at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:293)
   [junit4]    > at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:236)
   [junit4]    > at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
   [junit4]    > at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:69)
   [junit4]    > at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:645)
   [junit4]    > at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:72)
   [junit4]    > at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:449)
   [junit4]    > at org.apache.lucene.search.TestSimilarity.testSimilarity(TestSimilarity.java:137)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=PostingsFormat(name=LuceneFixedGap)}, docValues:{}, maxPointsInLeafNode=847, maxMBSortInHeap=6.087526796084562, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@7335a5b1), locale=es-AR, timezone=America/Indianapolis
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=88652784,total=294649856
   [junit4]   2> NOTE: All tests run in this JVM: [TestMixedDocValuesUpdates, TestSingleInstanceLockFactory, TestPositiveScoresOnlyCollector, Test2BPostingsBytes, TestDocValuesQueries, TestParallelTermEnum, TestReusableStringReader, TestIndexWriterUnicode, TestAxiomaticF1EXP, TestPrefixRandom, TestTermsEnum2, TestRAMDirectory, TestLucene50StoredFieldsFormat, TestIntroSorter, TestGraphTokenizers, TestSortedNumericSortField, TestBlendedTermQuery, TestWildcard, TestMatchNoDocsQuery, TestTimSorter, TestFileSwitchDirectory, TestNoMergeScheduler, TestBufferedChecksum, TestNorms, TestDuelingCodecsAtNight, TestAxiomaticF2LOG, TestStressNRT, TestTimeLimitingCollector, TestAxiomaticSimilarity, TestDeterminism, TestDocumentsWriterDeleteQueue, TestLevenshteinAutomata, TestCloseableThreadLocal, TestCheckIndex, TestFieldInvertState, TestToken, TestConcurrentMergeScheduler, TestIndexFileDeleter, TestPolygon, TestBoostQuery, TestElevationComparator, TestNumericDocValuesUpdates, TestMaxPosition, TestWeakIdentityMap, TestFilterDirectoryReader, TestSPIClassIterator, TestTragicIndexWriterDeadlock, TestRecyclingIntBlockAllocator, TestMultiDocValues, TestParallelLeafReader, TestMSBRadixSorter, TestIndexOrDocValuesQuery, TestRollback, Test2BPostings, TestBasicModelD, TestMergeSchedulerExternal, TestBooleanOr, TestLucene70DocValuesFormat, TestTieredMergePolicy, TestMutableValues, TestRateLimiter, TestRecyclingByteBlockAllocator, TestFilterSpans, TestOmitPositions, TestRegexpRandom2, TestDocInverterPerFieldErrorInfo, TestTermdocPerf, TestFieldValueQuery, TestPrefixInBooleanQuery, TestCrash, TestInPlaceMergeSorter, TestTransactionRollback, TestAddIndexes, TestLockFactory, TestMinimize, TestPriorityQueue, TestIndexWriterOutOfFileDescriptors, TestIndexWriterDelete, Test2BSortedDocValuesFixedSorted, TestIndexWriterWithThreads, TestSearcherManager, TestDeletionPolicy, TestIndexWriterMergePolicy, TestBoolean2, TestAtomicUpdate, TestStressAdvance, TestIndexWriterCommit, TestDirectoryReaderReopen, TestSearchAfter, TestHighCompressionMode, TestDocumentsWriterStallControl, TestTopDocsMerge, TestLongPostings, TestPerFieldPostingsFormat, TestPhraseQuery, TestBagOfPositions, TestOmitNorms, TestComplexExplanations, TestSegmentReader, TestFastDecompressionMode, TestTransactions, TestSimpleSearchEquivalence, TestCustomSearcherSort, TestHugeRamFile, TestSpanExplanations, TestStressIndexing2, TestCodecs, TestThreadedForceMerge, TestDisjunctionMaxQuery, TestOmitTf, TestSameScoresWithThreads, TestMultiTermConstantScore, TestForTooMuchCloning, TestDoc, TestExternalCodecs, TestTermVectorsReader, TestPerSegmentDeletes, TestBooleanQuery, TestComplexExplanationsOfNonMatches, TestUnicodeUtil, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestParallelReaderEmptyIndex, TestDocument, TestSumDocFreq, TestIndexableField, TestSentinelIntSet, TestRegexpRandom, TestPayloadsOnVectors, TestSizeBoundedForceMerge, TestCompiledAutomaton, TestSubScorerFreqs, TestIndexWriterConfig, TestMultiTermQueryRewrites, TestPositionIncrement, TestSimilarity]
   [junit4] Completed [318/475 (2!)] on J1 in 0.26s, 1 test, 1 failure <<< FAILURES!

[...truncated 11620 lines...]
   [junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
   [junit4]   2> 2965351 INFO  (SUITE-ForceLeaderTest-seed#[228BDCE02D54C240]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/init-core-data-001
   [junit4]   2> 2965368 INFO  (SUITE-ForceLeaderTest-seed#[228BDCE02D54C240]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2965369 INFO  (SUITE-ForceLeaderTest-seed#[228BDCE02D54C240]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 2965369 INFO  (SUITE-ForceLeaderTest-seed#[228BDCE02D54C240]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 2965371 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2965371 INFO  (Thread-2575) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2965371 INFO  (Thread-2575) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2965406 ERROR (Thread-2575) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2965471 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkTestServer start zk server on port:36710
   [junit4]   2> 2965512 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2965513 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2965514 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2965527 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2965527 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2965528 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2965528 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2965529 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2965530 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2965530 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2965547 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2965548 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 2965996 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2966031 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56b032cc{/,null,AVAILABLE}
   [junit4]   2> 2966032 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@328fc2ca{HTTP/1.1,[http/1.1]}{127.0.0.1:44470}
   [junit4]   2> 2966032 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server Started @2991596ms
   [junit4]   2> 2966032 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=45413, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/control-001/cores}
   [junit4]   2> 2966032 ERROR (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2966039 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2966039 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2966039 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2966039 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T02:43:13.490Z
   [junit4]   2> 2966041 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2966041 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/control-001/solr.xml
   [junit4]   2> 2966044 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2966044 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2966045 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2966060 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36710/solr
   [junit4]   2> 2966332 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2966333 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45413_
   [junit4]   2> 2966333 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.c.Overseer Overseer (id=98977827574448132-127.0.0.1:45413_-n_0000000000) starting
   [junit4]   2> 2966369 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45413_
   [junit4]   2> 2966385 INFO  (zkCallback-2731-thread-1-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2966616 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2966649 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2966649 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2966650 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:45413_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/control-001/cores
   [junit4]   2> 2966702 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2966703 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36710/solr ready
   [junit4]   2> 2966703 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50120,localport=45413], receiveBufferSize:531000
   [junit4]   2> 2966703 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44470,localport=59730], receiveBufferSize=530904
   [junit4]   2> 2966704 INFO  (qtp2003783633-14265) [n:127.0.0.1:45413_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45413_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2966714 INFO  (OverseerThreadFactory-3633-thread-1-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 2966818 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50124,localport=45413], receiveBufferSize:531000
   [junit4]   2> 2966818 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44470,localport=59734], receiveBufferSize=530904
   [junit4]   2> 2966819 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2966819 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2966934 INFO  (zkCallback-2731-thread-1-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2968037 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2968105 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2968526 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2968580 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 2968581 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2968581 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2968581 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2968600 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7009495157606532]
   [junit4]   2> 2968621 WARN  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2968836 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2968836 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2968837 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2968837 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2968838 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=74.21875, floorSegmentMB=0.234375, forceMergeDeletesPctAllowed=18.37321507942112, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1384313433698579
   [junit4]   2> 2968858 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@45987b3e[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2968872 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2968872 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2968872 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2968893 INFO  (searcherExecutor-3636-thread-1-processing-n:127.0.0.1:45413_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@45987b3e[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2968893 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583645244883206144
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45413/control_collection_shard1_replica_n1/
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45413/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2968909 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2968911 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45413/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2968942 INFO  (zkCallback-2731-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2968965 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2969014 INFO  (qtp2003783633-14263) [n:127.0.0.1:45413_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2196
   [junit4]   2> 2969016 INFO  (qtp2003783633-14265) [n:127.0.0.1:45413_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2969125 INFO  (zkCallback-2731-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2970022 INFO  (qtp2003783633-14265) [n:127.0.0.1:45413_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45413_&wt=javabin&version=2} status=0 QTime=3317
   [junit4]   2> 2970060 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2970075 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36710/solr ready
   [junit4]   2> 2970075 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2970079 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50136,localport=45413], receiveBufferSize:531000
   [junit4]   2> 2970116 INFO  (qtp2003783633-14266) [n:127.0.0.1:45413_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2970116 INFO  (SocketProxy-Acceptor-45413) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44470,localport=59746], receiveBufferSize=530904
   [junit4]   2> 2970172 INFO  (OverseerThreadFactory-3633-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2970205 INFO  (OverseerCollectionConfigSetProcessor-98977827574448132-127.0.0.1:45413_-n_0000000000) [n:127.0.0.1:45413_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2970205 WARN  (OverseerThreadFactory-3633-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 2970320 INFO  (qtp2003783633-14266) [n:127.0.0.1:45413_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2970320 INFO  (qtp2003783633-14266) [n:127.0.0.1:45413_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=204
   [junit4]   2> 2970670 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001 of type TLOG
   [junit4]   2> 2970671 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@68da6b14{/,null,AVAILABLE}
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@135e4712{HTTP/1.1,[http/1.1]}{127.0.0.1:42613}
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server Started @2996236ms
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=37340, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001/cores}
   [junit4]   2> 2970672 ERROR (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2970672 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T02:43:18.123Z
   [junit4]   2> 2970700 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2970700 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001/solr.xml
   [junit4]   2> 2970735 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2970735 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2970736 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2970751 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36710/solr
   [junit4]   2> 2970805 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2970832 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2970834 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37340_
   [junit4]   2> 2970835 INFO  (zkCallback-2739-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2970909 INFO  (zkCallback-2744-thread-1-processing-n:127.0.0.1:37340_) [n:127.0.0.1:37340_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2970835 INFO  (zkCallback-2731-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2971217 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2971264 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2971264 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2971265 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:37340_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001/cores
   [junit4]   2> 2971331 INFO  (SocketProxy-Acceptor-37340) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54890,localport=37340], receiveBufferSize:531000
   [junit4]   2> 2971355 INFO  (SocketProxy-Acceptor-37340) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=42613,localport=33462], receiveBufferSize=530904
   [junit4]   2> 2971369 INFO  (qtp2081090891-14319) [n:127.0.0.1:37340_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37340_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2971370 INFO  (OverseerCollectionConfigSetProcessor-98977827574448132-127.0.0.1:45413_-n_0000000000) [n:127.0.0.1:45413_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2971371 INFO  (OverseerThreadFactory-3633-thread-3-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:37340_ for creating new replica
   [junit4]   2> 2971381 INFO  (SocketProxy-Acceptor-37340) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54898,localport=37340], receiveBufferSize:531000
   [junit4]   2> 2971381 INFO  (SocketProxy-Acceptor-37340) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=42613,localport=33470], receiveBufferSize=530904
   [junit4]   2> 2971381 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2971381 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2972468 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2972525 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
   [junit4]   2> 2973021 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2973060 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
   [junit4]   2> 2973060 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2973060 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2973060 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
   [junit4]   2> 2973079 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7009495157606532]
   [junit4]   2> 2973095 WARN  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2973243 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2973243 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2973260 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2973260 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2973261 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=74.21875, floorSegmentMB=0.234375, forceMergeDeletesPctAllowed=18.37321507942112, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1384313433698579
   [junit4]   2> 2973293 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f547567[collection1_shard2_replica_t41] main]
   [junit4]   2> 2973297 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2973297 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2973297 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2973298 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583645249502183424
   [junit4]   2> 2973315 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2973315 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2973315 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37340/collection1_shard2_replica_t41/
   [junit4]   2> 2973315 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2973316 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:37340/collection1_shard2_replica_t41/ has no replicas
   [junit4]   2> 2973316 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2973316 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
   [junit4]   2> 2973350 INFO  (searcherExecutor-3647-thread-1-processing-n:127.0.0.1:37340_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@1f547567[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2973351 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37340/collection1_shard2_replica_t41/ shard2
   [junit4]   2> 2973589 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2973716 INFO  (qtp2081090891-14317) [n:127.0.0.1:37340_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2335
   [junit4]   2> 2973718 INFO  (qtp2081090891-14319) [n:127.0.0.1:37340_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:37340_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2348
   [junit4]   2> 2974066 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001 of type TLOG
   [junit4]   2> 2974082 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2974107 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44e192f6{/,null,AVAILABLE}
   [junit4]   2> 2974107 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@53d18742{HTTP/1.1,[http/1.1]}{127.0.0.1:34280}
   [junit4]   2> 2974107 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server Started @2999671ms
   [junit4]   2> 2974107 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=39970, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001/cores}
   [junit4]   2> 2974107 ERROR (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2974131 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2974131 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2974131 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2974131 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T02:43:21.582Z
   [junit4]   2> 2974133 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2974133 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001/solr.xml
   [junit4]   2> 2974144 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2974144 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2974145 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2974160 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36710/solr
   [junit4]   2> 2974174 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2974184 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2974185 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39970_
   [junit4]   2> 2974199 INFO  (zkCallback-2731-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2974199 INFO  (zkCallback-2744-thread-1-processing-n:127.0.0.1:37340_) [n:127.0.0.1:37340_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2974199 INFO  (zkCallback-2739-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2974206 INFO  (zkCallback-2750-thread-1-processing-n:127.0.0.1:39970_) [n:127.0.0.1:39970_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2974352 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2974386 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2974386 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2974399 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:39970_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001/cores
   [junit4]   2> 2974501 INFO  (qtp2081090891-14320) [n:127.0.0.1:37340_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39970_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2974507 INFO  (OverseerCollectionConfigSetProcessor-98977827574448132-127.0.0.1:45413_-n_0000000000) [n:127.0.0.1:45413_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2974508 INFO  (OverseerThreadFactory-3633-thread-4-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:39970_ for creating new replica
   [junit4]   2> 2974509 INFO  (SocketProxy-Acceptor-39970) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48822,localport=39970], receiveBufferSize:531000
   [junit4]   2> 2974538 INFO  (SocketProxy-Acceptor-39970) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=34280,localport=34774], receiveBufferSize=530904
   [junit4]   2> 2974539 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2974539 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2975572 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2975617 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
   [junit4]   2> 2975963 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2976002 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
   [junit4]   2> 2976002 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2976002 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2976003 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
   [junit4]   2> 2976013 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7009495157606532]
   [junit4]   2> 2976046 WARN  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2976199 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2976199 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2976213 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2976213 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2976214 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=74.21875, floorSegmentMB=0.234375, forceMergeDeletesPctAllowed=18.37321507942112, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1384313433698579
   [junit4]   2> 2976214 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@17488018[collection1_shard1_replica_t43] main]
   [junit4]   2> 2976215 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2976215 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2976215 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2976216 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583645252561928192
   [junit4]   2> 2976250 INFO  (searcherExecutor-3658-thread-1-processing-n:127.0.0.1:39970_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@17488018[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:39970/collection1_shard1_replica_t43/
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:39970/collection1_shard1_replica_t43/ has no replicas
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2976266 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
   [junit4]   2> 2976268 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:39970/collection1_shard1_replica_t43/ shard1
   [junit4]   2> 2976391 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2976393 INFO  (qtp1619022720-14353) [n:127.0.0.1:39970_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1853
   [junit4]   2> 2976394 INFO  (qtp2081090891-14320) [n:127.0.0.1:37340_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:39970_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1893
   [junit4]   2> 2976526 INFO  (OverseerCollectionConfigSetProcessor-98977827574448132-127.0.0.1:45413_-n_0000000000) [n:127.0.0.1:45413_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2976757 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-3-001 of type TLOG
   [junit4]   2> 2976758 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2976759 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6eacaf95{/,null,AVAILABLE}
   [junit4]   2> 2976759 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@13d9256b{HTTP/1.1,[http/1.1]}{127.0.0.1:38256}
   [junit4]   2> 2976760 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.e.j.s.Server Started @3002324ms
   [junit4]   2> 2976760 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=36331, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-3-001/cores}
   [junit4]   2> 2976760 ERROR (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2976761 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2976761 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2976761 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2976761 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T02:43:24.212Z
   [junit4]   2> 2976783 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2976783 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001/shard-3-001/solr.xml
   [junit4]   2> 2976786 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2976786 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2976787 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2976790 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36710/solr
   [junit4]   2> 2976820 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:36331_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2976822 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:36331_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2976832 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:36331_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36331_
   [junit4]   2> 2976833 INFO  (zkCallback-2750-thread-1-processing-n:127.0.0.1:39970_) [n:127.0.0.1:39970_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2976834 INFO  (zkCallback-2739-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2976833 INFO  (zkCallback-2744-thread-1-processing-n:127.0.0.1:37340_) [n:127.0.0.1:37340_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2976834 INFO  (zkCallback-2731-thread-2-processing-n:127.0.0.1:45413_) [n:127.0.0.1:45413_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2976900 INFO  (zkCallback-2756-thread-1-processing-n:127.0.0.1:36331_) [n:127.0.0.1:36331_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2977173 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:36331_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2977211 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[228BDCE02D54C240]) [n:127.0.0.1:36331_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21e1e700
   [junit4]   2> 2977211 INFO  (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-see

[...truncated too long message...]

e-seed#[228BDCE02D54C240]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 3754595 INFO  (zkCallback-2800-thread-2-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 3754595 INFO  (zkCallback-2783-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 3754638 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.Overseer Overseer (id=98977874499928079-127.0.0.1:43017_-n_0000000003) closing
   [junit4]   2> 3754642 INFO  (OverseerStateUpdate-98977874499928079-127.0.0.1:43017_-n_0000000003) [n:127.0.0.1:43017_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43017_
   [junit4]   2> 3754642 WARN  (OverseerAutoScalingTriggerThread-98977874499928079-127.0.0.1:43017_-n_0000000003) [n:127.0.0.1:43017_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3756440 WARN  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 3756440 INFO  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 3756440 INFO  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [collection1_shard2_replica_n45]  CLOSING SolrCore org.apache.solr.core.SolrCore@7437f550
   [junit4]   2> 3756440 INFO  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard2.replica_n45, tag=1949824336
   [junit4]   2> 3756440 INFO  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@30e8ecb7: rootName = null, domain = solr.core.collection1.shard2.replica_n45, service url = null, agent id = null] for registry solr.core.collection1.shard2.replica_n45 / com.codahale.metrics.MetricRegistry@ba2d94
   [junit4]   2> 3756564 INFO  (zkCallback-2800-thread-1-processing-n:127.0.0.1:43017_) [n:127.0.0.1:43017_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=1949824336
   [junit4]   2> 3756623 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@1127a0b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 3756624 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@663690bc{/,null,UNAVAILABLE}
   [junit4]   2> 3756672 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ChaosMonkey monkey: stop jetty! 42092
   [junit4]   2> 3756672 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3756672 INFO  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45636 45636
   [junit4]   2> 3756806 INFO  (Thread-2625) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45636 45636
   [junit4]   2> 3756834 WARN  (Thread-2625) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/aliases.json
   [junit4]   2> 4 /solr/security.json
   [junit4]   2> 4 /solr/configs/conf1
   [junit4]   2> 3 /solr/collections/forceleader_last_published/state.json
   [junit4]   2> 3 /solr/collections/collection1/state.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/clusterstate.json
   [junit4]   2> 6 /solr/clusterprops.json
   [junit4]   2> 4 /solr/autoscaling.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 6 /solr/live_nodes
   [junit4]   2> 6 /solr/collections
   [junit4]   2> 4 /solr/overseer/queue
   [junit4]   2> 4 /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 4 /solr/overseer/queue-work
   [junit4]   2> 4 /solr/overseer/collection-queue-work
   [junit4]   2>
   [junit4]   2> 3756835 WARN  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SocketProxy Closing 2 connections to: http://127.0.0.1:43017/, target: http://127.0.0.1:35660/
   [junit4]   2> 3756835 WARN  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SocketProxy Closing 6 connections to: http://127.0.0.1:42092/, target: http://127.0.0.1:43866/
   [junit4]   2> 3756835 WARN  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:34696/, target: http://127.0.0.1:41852/
   [junit4]   2> 3756835 WARN  (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[228BDCE02D54C240]) [    ] o.a.s.c.SocketProxy Closing 2 connections to: http://127.0.0.1:36032/, target: http://127.0.0.1:40115/
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_228BDCE02D54C240-001
   [junit4]   2> Nov 10, 2017 2:56:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=FSTOrd50, a_t=FST50, id=PostingsFormat(name=Memory), text=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1099, maxMBSortInHeap=7.152259224115535, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2b726c9b), locale=ar-LB, timezone=Pacific/Kosrae
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=231656432,total=526909440
   [junit4]   2> NOTE: All tests run in this JVM: [TestFieldCacheSortRandom, TestRawResponseWriter, CoreAdminCreateDiscoverTest, TestNamedUpdateProcessors, DistributedMLTComponentTest, TestSolrJ, TestRequestForwarding, TestUseDocValuesAsStored2, CursorPagingTest, SolrCoreTest, TestBinaryField, TestRestoreCore, StressHdfsTest, DocValuesTest, TestUniqueKeyFieldResource, PKIAuthenticationIntegrationTest, MinimalSchemaTest, TestXmlQParserPlugin, TestCustomDocTransformer, TestBulkSchemaConcurrent, TestSolrCoreParser, ResponseLogComponentTest, SecurityConfHandlerTest, TestImplicitCoreProperties, TestApiFramework, TestCollapseQParserPlugin, ResourceLoaderTest, TestCSVLoader, RecoveryAfterSoftCommitTest, TestReversedWildcardFilterFactory, ActionThrottleTest, TestCollationFieldDocValues, UpdateLogTest, CdcrReplicationDistributedZkTest, TestUpdate, TestCopyFieldCollectionResource, AnalysisAfterCoreReloadTest, TestHdfsBackupRestoreCore, ZkStateWriterTest, CdcrVersionReplicationTest, TestJsonRequest, TestStressVersions, OverseerStatusTest, RankQueryTest, JSONWriterTest, ReplicationFactorTest, TestMiniSolrCloudClusterSSL, JavabinLoaderTest, SpatialFilterTest, PreAnalyzedFieldManagedSchemaCloudTest, UpdateRequestProcessorFactoryTest, TestTestInjection, TestRuleBasedAuthorizationPlugin, DistribCursorPagingTest, TestSimpleTextCodec, BaseCdcrDistributedZkTest, ClassificationUpdateProcessorIntegrationTest, JsonLoaderTest, TestSchemaSimilarityResource, TestIndexingPerformance, CursorMarkTest, TestUtils, HLLUtilTest, TestSchemaManager, TestHighlightDedupGrouping, DeleteReplicaTest, FastVectorHighlighterTest, SoftAutoCommitTest, UpdateParamsTest, VMParamsZkACLAndCredentialsProvidersTest, TestTolerantUpdateProcessorRandomCloud, TestConfig, TestJoin, DateMathParserTest, ExecutePlanActionTest, TestStressLiveNodes, CachingDirectoryFactoryTest, TestNestedDocsSort, HdfsNNFailoverTest, TestSegmentSorting, LeaderInitiatedRecoveryOnCommitTest, CollectionStateFormat2Test, PeerSyncTest, TestObjectReleaseTracker, RulesTest, HdfsDirectoryTest, DocumentBuilderTest, CacheHeaderTest, PrimitiveFieldTypeTest, TestWordDelimiterFilterFactory, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestFoldingMultitermQuery, TestLRUStatsCache, TestCloudSearcherWarming, QueryParsingTest, TestFileDictionaryLookup, TestDistribIDF, ConnectionManagerTest, TestComponentsName, DistanceUnitsTest, TestRandomDVFaceting, TestSweetSpotSimilarityFactory, TestSolrIndexConfig, HdfsChaosMonkeyNothingIsSafeTest, ZkControllerTest, TestConfigSetsAPIZkFailure, DeleteInactiveReplicaTest, SpellCheckCollatorWithCollapseTest, NoCacheHeaderTest, TestHttpShardHandlerFactory, ForceLeaderTest]
   [junit4] Completed [370/749 (1!)] on J0 in 792.31s, 3 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 48478 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2168 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2168/

No tests ran.

Build Log:
[...truncated 12088 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_D8117D99600FAF63-001/init-core-data-001
   [junit4]   2> 2088450 WARN  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=77 numCloses=77
   [junit4]   2> 2088450 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2088451 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2088451 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2088452 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_D8117D99600FAF63-001/tempDir-001
   [junit4]   2> 2088452 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2088462 INFO  (Thread-1365) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2088462 INFO  (Thread-1365) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2088499 ERROR (Thread-1365) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2088583 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:45466
   [junit4]   2> 2088722 INFO  (jetty-launcher-2377-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2088775 INFO  (jetty-launcher-2377-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25bb820d{/solr,null,AVAILABLE}
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@73616def{HTTP/1.1,[http/1.1]}{127.0.0.1:35110}
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.e.j.s.Server Started @2104118ms
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35110}
   [junit4]   2> 2088776 ERROR (jetty-launcher-2377-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2088776 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T21:01:00.930Z
   [junit4]   2> 2088812 INFO  (jetty-launcher-2377-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2088880 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2088919 INFO  (jetty-launcher-2377-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45466/solr
   [junit4]   2> 2088953 INFO  (jetty-launcher-2377-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@69d7a603{/solr,null,AVAILABLE}
   [junit4]   2> 2089027 INFO  (jetty-launcher-2377-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@185a2297{HTTP/1.1,[http/1.1]}{127.0.0.1:33769}
   [junit4]   2> 2089027 INFO  (jetty-launcher-2377-thread-2) [    ] o.e.j.s.Server Started @2104369ms
   [junit4]   2> 2089027 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33769}
   [junit4]   2> 2089028 ERROR (jetty-launcher-2377-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2089028 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2089028 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2089028 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2089028 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-10T21:01:01.182Z
   [junit4]   2> 2089052 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2089579 INFO  (jetty-launcher-2377-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45466/solr
   [junit4]   2> 2089945 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2089945 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33769_solr
   [junit4]   2> 2089946 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.c.Overseer Overseer (id=98982144286130182-127.0.0.1:33769_solr-n_0000000000) starting
   [junit4]   2> 2090405 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33769_solr
   [junit4]   2> 2090421 INFO  (zkCallback-2390-thread-1-processing-n:127.0.0.1:33769_solr) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2092003 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33769.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2092351 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33769.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2092352 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33769.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2092353 INFO  (jetty-launcher-2377-thread-2) [n:127.0.0.1:33769_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_D8117D99600FAF63-001/tempDir-001/node2/.
   [junit4]   2> 2092442 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2092461 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2092479 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35110_solr
   [junit4]   2> 2092480 INFO  (zkCallback-2390-thread-1-processing-n:127.0.0.1:33769_solr) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2092627 INFO  (zkCallback-2385-thread-1-processing-n:127.0.0.1:35110_solr) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2093525 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35110.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2093562 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35110.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2093631 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35110.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a0fe74
   [junit4]   2> 2093632 INFO  (jetty-launcher-2377-thread-1) [n:127.0.0.1:35110_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_D8117D99600FAF63-001/tempDir-001/node1/.
   [junit4]   2> 2094129 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2094129 INFO  (SUITE-AutoScalingHandlerTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2094256 INFO  (TEST-AutoScalingHandlerTest.testPolicyAndPreferences-seed#[D8117D99600FAF63]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testPolicyAndPreferences
   [junit4]   2> 2094330 ERROR (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, errors: [{set-policy={xyz=[{replica=<2, shard=#EACH, node=#ANY}, {nodeRole=!overseer, replica=0}], policy1=[{cores=<2, node=#ANY}, {replica=<2, shard=#EACH, node=#ANY}]}, errorMessages=[cores is only allowed in 'cluster-policy']}],
   [junit4]   2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.processOps(AutoScalingHandler.java:211)
   [junit4]   2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.handleRequestBody(AutoScalingHandler.java:145)
   [junit4]   2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:737)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:718)
   [junit4]   2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:499)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 2094330 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=400 QTime=1
   [junit4]   2> 2094421 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2094421 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2094423 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2094424 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2094548 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=147
   [junit4]   2> 2094677 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2094677 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2094678 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2094912 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2094932 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=365
   [junit4]   2> 2095058 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2095058 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2095115 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2095131 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2095205 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=271
   [junit4]   2> 2095371 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2095372 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2095388 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2095389 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 2095409 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=203
   [junit4]   2> 2095462 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2095498 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2095500 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm} status=0 QTime=0
   [junit4]   2> 2095501 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm} status=0 QTime=0
   [junit4]   2> 2095535 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=124
   [junit4]   2> 2095609 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2095609 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2095610 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName,os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm,solr.core} status=0 QTime=0
   [junit4]   2> 2095644 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName,os.systemLoadAverage&wt=javabin&version=2&group=solr.jvm,solr.core} status=0 QTime=0
   [junit4]   2> 2095646 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=109
   [junit4]   2> 2095646 INFO  (TEST-AutoScalingHandlerTest.testPolicyAndPreferences-seed#[D8117D99600FAF63]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testPolicyAndPreferences
   [junit4]   2> 2095903 INFO  (TEST-AutoScalingHandlerTest.testConcurrentUpdates-seed#[D8117D99600FAF63]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentUpdates
   [junit4]   2> 2096632 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2096633 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2096633 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096643 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096725 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2096726 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2096726 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096727 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096778 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=494
   [junit4]   2> 2096957 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2096957 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2096958 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096958 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2096994 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=215
   [junit4]   2> 2097046 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097047 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097047 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097103 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097156 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097157 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097157 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097158 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097173 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=178
   [junit4]   2> 2097193 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097193 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097194 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097229 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097230 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097231 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097231 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097232 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097246 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=72
   [junit4]   2> 2097266 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097266 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097282 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097283 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097304 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097305 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097305 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097318 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097319 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097319 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097320 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097321 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=1116
   [junit4]   2> 2097375 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=69
   [junit4]   2> 2097534 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097534 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097535 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097535 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097538 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097538 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097570 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097595 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097647 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=236
   [junit4]   2> 2097698 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097699 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097699 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097704 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097717 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097718 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097720 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097721 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097879 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=211
   [junit4]   2> 2097952 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2097953 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2097953 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097954 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2097989 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=706
   [junit4]   2> 2098010 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098010 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098011 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098011 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098026 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098028 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098028 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098029 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098063 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098099 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=109
   [junit4]   2> 2098099 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098100 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098100 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098267 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098267 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098268 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098322 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098322 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=53
   [junit4]   2> 2098330 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098331 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=232
   [junit4]   2> 2098358 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098359 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098396 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098453 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098469 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098470 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098471 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098471 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098525 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098526 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098527 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=631
   [junit4]   2> 2098541 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa7bc5d480023, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2098769 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098830 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2098842 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2098842 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098843 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2098878 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=546
   [junit4]   2> 2098906 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099013 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099014 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099014 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099015 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099016 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099016 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099016 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=137
   [junit4]   2> 2099066 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099067 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099067 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099068 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099068 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=18
   [junit4]   2> 2099126 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099140 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099141 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099142 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099143 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099143 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099143 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099144 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099159 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=602
   [junit4]   2> 2099175 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099176 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099176 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099196 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099212 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099212 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099213 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099216 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099232 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa7bc5d48002c, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2099233 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=73
   [junit4]   2> 2099319 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099321 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099321 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099338 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099379 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=145
   [junit4]   2> 2099381 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099382 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099382 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099395 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099445 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099446 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099452 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099452 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099525 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=145
   [junit4]   2> 2099761 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099761 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099762 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099763 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099763 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099763 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099764 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099764 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099809 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=255
   [junit4]   2> 2099979 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2099992 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2099993 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099993 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2099994 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=925
   [junit4]   2> 2100013 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100014 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100017 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100017 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100139 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100140 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100140 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100141 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100175 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=180
   [junit4]   2> 2100273 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100274 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100274 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100275 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100286 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100286 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100287 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100287 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100289 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=112
   [junit4]   2> 2100543 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100543 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100544 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100544 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100544 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100545 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100562 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100594 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=272
   [junit4]   2> 2100614 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100652 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100652 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100652 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100653 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100726 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=131
   [junit4]   2> 2100742 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100742 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100743 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100743 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100778 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100779 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100779 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100780 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100780 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa7bc5d48003b, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2100814 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100815 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100815 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100815 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100817 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=1007
   [junit4]   2> 2100871 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100872 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100872 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100873 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100889 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=72
   [junit4]   2> 2100907 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100908 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100909 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100909 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100909 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa7bc5d48003e, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2100983 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100984 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100984 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2100997 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2100998 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2100999 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101000 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101034 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=307
   [junit4]   2> 2101041 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101071 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101128 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101129 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101129 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101129 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101130 INFO  (qtp415266916-7342) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=222
   [junit4]   2> 2101151 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101151 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101163 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101291 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101292 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101495 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=202
   [junit4]   2> 2101496 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101496 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101509 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101511 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=380
   [junit4]   2> 2101517 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101518 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101648 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101649 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101649 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101650 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101730 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=218
   [junit4]   2> 2101811 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101811 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101812 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101812 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101847 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101848 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101848 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101849 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101883 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=153
   [junit4]   2> 2101981 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2101982 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2101982 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2101983 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102055 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102055 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102056 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102056 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102078 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=1043
   [junit4]   2> 2102116 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102117 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102117 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102127 INFO  (qtp415266916-7344) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102136 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102152 INFO  (qtp227244561-7334) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=73
   [junit4]   2> 2102192 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102193 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102200 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102290 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102291 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102291 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102292 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102310 INFO  (qtp415266916-7351) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=158
   [junit4]   2> 2102384 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102384 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102385 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102385 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102385 INFO  (qtp227244561-7329) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102385 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102385 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102386 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102387 INFO  (qtp415266916-7341) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=75
   [junit4]   2> 2102494 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102494 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102495 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102495 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102558 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=171
   [junit4]   2> 2102619 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102692 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102692 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102693 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102721 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102721 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102722 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102722 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102723 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=156
   [junit4]   2> 2102794 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102794 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102794 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102795 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102866 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=142
   [junit4]   2> 2102911 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2102984 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2102984 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2102985 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103030 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103031 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103037 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103038 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103093 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=207
   [junit4]   2> 2103200 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103202 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103203 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103240 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103241 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103242 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103256 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103258 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103331 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103332 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103332 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103338 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=207
   [junit4]   2> 2103346 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103388 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103389 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103389 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103390 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103403 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103403 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fa7bc5d480058, likely client has closed socket
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2103404 INFO  (qtp227244561-7333) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=56
   [junit4]   2> 2103423 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103439 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103440 INFO  (qtp415266916-7345) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103442 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103443 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103443 INFO  (qtp227244561-7336) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103476 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103515 INFO  (qtp227244561-7332) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=110
   [junit4]   2> 2103549 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103550 INFO  (qtp227244561-7335) [n:127.0.0.1:35110_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45466/solr ready
   [junit4]   2> 2103558 INFO  (qtp227244561-7330) [n:127.0.0.1:35110_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103569 INFO  (qtp415266916-7346) [n:127.0.0.1:33769_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 2103680 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2103680 INFO  (qtp415266916-7352) [n:127.0.0.1:33769_solr    ] o.a.s.c.s.i

[...truncated too long message...]

-thread-4-processing-n:127.0.0.1:38663_solr) [n:127.0.0.1:38663_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 5297305 INFO  (zkCallback-2336-thread-5-processing-n:127.0.0.1:38469_solr) [n:127.0.0.1:38469_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 5297305 INFO  (zkCallback-2336-thread-6-processing-n:127.0.0.1:38469_solr) [n:127.0.0.1:38469_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testErrorChecks-collection/state.json] for collection [testErrorChecks-collection] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 5297375 INFO  (coreCloseExecutor-4727-thread-1) [n:127.0.0.1:38663_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=597154821
   [junit4]   2> 5297411 INFO  (coreCloseExecutor-4727-thread-3) [n:127.0.0.1:38663_solr c:testErrorChecks-collection s:shard2 r:core_node4 x:testErrorChecks-collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testErrorChecks-collection.shard2.leader, tag=42003991
   [junit4]   2> 5297501 INFO  (coreCloseExecutor-4728-thread-2) [n:127.0.0.1:38469_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testErrorChecks-collection.shard1.replica_n1, tag=1379917740
   [junit4]   2> 5297556 INFO  (coreCloseExecutor-4728-thread-2) [n:127.0.0.1:38469_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2619a3d0: rootName = solr_38469, domain = solr.core.testErrorChecks-collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testErrorChecks-collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@9d462b2
   [junit4]   2> 5297612 INFO  (coreCloseExecutor-4728-thread-2) [n:127.0.0.1:38469_solr c:testErrorChecks-collection s:shard1 r:core_node3 x:testErrorChecks-collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testErrorChecks-collection.shard1.leader, tag=1379917740
   [junit4]   2> 5297642 INFO  (jetty-closer-2329-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6e931f5c{/solr,null,UNAVAILABLE}
   [junit4]   2> 5297661 INFO  (coreCloseExecutor-4728-thread-1) [n:127.0.0.1:38469_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=290126932
   [junit4]   2> 5297706 INFO  (jetty-closer-2329-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98982351461089294-127.0.0.1:38469_solr-n_0000000002) closing
   [junit4]   2> 5297706 WARN  (OverseerAutoScalingTriggerThread-98982351461089294-127.0.0.1:38469_solr-n_0000000002) [n:127.0.0.1:38469_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 5297949 INFO  (OverseerStateUpdate-98982351461089294-127.0.0.1:38469_solr-n_0000000002) [n:127.0.0.1:38469_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:38469_solr
   [junit4]   2> 5298015 INFO  (jetty-closer-2329-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@48d7bd38{/solr,null,UNAVAILABLE}
   [junit4]   2> 5298016 ERROR (SUITE-AliasIntegrationTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 5298031 INFO  (SUITE-AliasIntegrationTest-seed#[D8117D99600FAF63]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43638 43638
   [junit4]   2> 5298140 INFO  (Thread-2992) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43638 43638
   [junit4]   2> 5298141 WARN  (Thread-2992) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 11 /solr/clusterprops.json
   [junit4]   2> 11 /solr/aliases.json
   [junit4]   2> 4 /solr/security.json
   [junit4]   2> 4 /solr/configs/conf
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 19 /solr/collections/testErrorChecks-collection/state.json
   [junit4]   2> 17 /solr/collections/collection1/state.json
   [junit4]   2> 11 /solr/clusterstate.json
   [junit4]   2> 7 /solr/collections/collection2/state.json
   [junit4]   2> 2 /solr/autoscaling.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 11 /solr/live_nodes
   [junit4]   2> 11 /solr/collections
   [junit4]   2> 2 /solr/overseer/queue
   [junit4]   2> 2 /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 2 /solr/overseer/queue-work
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.AliasIntegrationTest_D8117D99600FAF63-001
   [junit4]   2> Nov 10, 2017 9:54:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {a_t=Lucene50(blocksize=128), id=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=88, maxMBSortInHeap=6.628891856072295, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@571d3160), locale=et-EE, timezone=Asia/Yakutsk
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=216090032,total=523239424
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedFacetPivotLongTailTest, TestMaxTokenLenTokenizer, TestExactStatsCache, TestPrepRecovery, TestConfigsApi, LegacyCloudClusterPropTest, TestConfigReload, ClusterStateTest, FullSolrCloudDistribCmdsTest, TestSQLHandlerNonCloud, TestReRankQParserPlugin, TestShardHandlerFactory, TemplateUpdateProcessorTest, TestPartialUpdateDeduplication, UnloadDistributedZkTest, TestSearcherReuse, TestEmbeddedSolrServerConstructors, TestDistributedGrouping, TestCoreDiscovery, SuggestComponentContextFilterQueryTest, DeleteNodeTest, TestSSLRandomization, HdfsCollectionsAPIDistributedZkTest, BasicDistributedZk2Test, XsltUpdateRequestHandlerTest, TestRandomFlRTGCloud, HLLSerializationTest, DirectSolrConnectionTest, TestReqParamsAPI, TestConfigSets, BitVectorTest, PeerSyncReplicationTest, TestFieldCacheWithThreads, HdfsWriteToMultipleCollectionsTest, TestStressInPlaceUpdates, OverseerTest, CollectionReloadTest, CustomCollectionTest, TestCloudSchemaless, AlternateDirectoryTest, ConjunctionSolrSpellCheckerTest, TestSerializedLuceneMatchVersion, CoreAdminHandlerTest, TestExpandComponent, DistanceFunctionTest, TestDelegationWithHadoopAuth, BasicAuthStandaloneTest, RegexBytesRefFilterTest, TestSolrCLIRunExample, TestQuerySenderListener, DocValuesNotIndexedTest, TestBinaryResponseWriter, TriggerIntegrationTest, MergeStrategyTest, SolrSlf4jReporterTest, TestMissingGroups, TestIntervalFaceting, TestPushWriter, TestScoreJoinQPScore, BadComponentTest, ManagedSchemaRoundRobinCloudTest, CSVRequestHandlerTest, TestStandardQParsers, TestManagedSynonymFilterFactory, TestConfigSetsAPIExclusivity, AddSchemaFieldsUpdateProcessorFactoryTest, HighlighterTest, TestHighFrequencyDictionaryFactory, SolrGraphiteReporterTest, TestSolrDeletionPolicy2, UUIDUpdateProcessorFallbackTest, QueryElevationComponentTest, TestNumericTerms64, TestConfigOverlay, AutoscalingHistoryHandlerTest, SolrCLIZkUtilsTest, BooleanFieldTest, HdfsRecoveryZkTest, TestExtendedDismaxParser, TestFunctionQuery, UniqFieldsUpdateProcessorFactoryTest, TestManagedResource, CoreMergeIndexesAdminHandlerTest, TestCloudJSONFacetJoinDomain, SimpleMLTQParserTest, SystemInfoHandlerTest, HdfsDirectoryFactoryTest, IndexBasedSpellCheckerTest, RuleEngineTest, RollingRestartTest, TestFieldCacheSort, TestLeaderElectionZkExpiry, DefaultValueUpdateProcessorTest, BufferStoreTest, CollectionsAPIAsyncDistributedZkTest, TestZkAclsWithHadoopAuth, CopyFieldTest, OverriddenZkACLAndCredentialsProvidersTest, RequiredFieldsTest, DistributedFacetPivotWhiteBoxTest, PropertiesRequestHandlerTest, FieldMutatingUpdateProcessorTest, TestTolerantUpdateProcessorCloud, MultiThreadedOCPTest, AnalysisErrorHandlingTest, TestNonDefinedSimilarityFactory, PeerSyncWithIndexFingerprintCachingTest, TestAuthenticationFramework, EchoParamsTest, TestSubQueryTransformer, AutoAddReplicasIntegrationTest, TestTolerantSearch, AliasIntegrationTest]
   [junit4] Completed [332/749 (4!)] on J2 in 48.61s, 2 tests, 1 failure <<< FAILURES!

[...truncated 468 lines...]
ERROR: command execution failed.
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-Tests-master #2168
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-Tests-master #2168
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2169 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2169/

6 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:36033/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36033/collection1
        at __randomizedtesting.SeedInfo.seed([B81B7FD6062BC52A:304F400CA8D7A8D2]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1582)
        at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=18803, name=zkCallback-1751-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=18766, name=zkCallback-1751-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=18809, name=zkCallback-1751-thread-4-SendThread(127.0.0.1:47527), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=18763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=18807, name=zkCallback-1751-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=18943, name=zkCallback-1751-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=18804, name=zkCallback-1751-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=18810, name=zkCallback-1751-thread-4-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
   1) Thread[id=18803, name=zkCallback-1751-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=18766, name=zkCallback-1751-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=18809, name=zkCallback-1751-thread-4-SendThread(127.0.0.1:47527), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=18763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=18807, name=zkCallback-1751-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=18943, name=zkCallback-1751-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=18804, name=zkCallback-1751-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=18810, name=zkCallback-1751-thread-4-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
        at __randomizedtesting.SeedInfo.seed([B81B7FD6062BC52A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=18803, name=zkCallback-1751-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=18766, name=zkCallback-1751-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=18809, name=zkCallback-1751-thread-4-SendThread(127.0.0.1:47527), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=18807, name=zkCallback-1751-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=18943, name=zkCallback-1751-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=18804, name=zkCallback-1751-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=18803, name=zkCallback-1751-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=18766, name=zkCallback-1751-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=18809, name=zkCallback-1751-thread-4-SendThread(127.0.0.1:47527), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=18807, name=zkCallback-1751-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=18943, name=zkCallback-1751-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=18804, name=zkCallback-1751-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([B81B7FD6062BC52A]:0)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication

Error Message:
Could not load collection from ZK: control_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: control_collection
        at __randomizedtesting.SeedInfo.seed([B81B7FD6062BC52A:F251EBD69783EA2F]:0)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
        at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:110)
        at org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication(ShardSplitTest.java:189)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/control_collection/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        ... 45 more


FAILED:  org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test

Error Message:
wrong number of events added to .system expected:<9> but was:<10>

Stack Trace:
java.lang.AssertionError: wrong number of events added to .system expected:<9> but was:<10>
        at __randomizedtesting.SeedInfo.seed([B81B7FD6062BC52A:304F400CA8D7A8D2]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test(SystemLogListenerTest.java:166)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRetryUpdatesWhenClusterStateIsStale

Error Message:
Error from server at http://127.0.0.1:46786/solr/stale_state_test_col_shard1_replica_n1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/stale_state_test_col_shard1_replica_n1/update. Reason: <pre>    Can not find: /solr/stale_state_test_col_shard1_replica_n1/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46786/solr/stale_state_test_col_shard1_replica_n1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/stale_state_test_col_shard1_replica_n1/update. Reason:
<pre>    Can not find: /solr/stale_state_test_col_shard1_replica_n1/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

        at __randomizedtesting.SeedInfo.seed([92915A1E74F774E0:26A0C2F6971E02CC]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:559)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRetryUpdatesWhenClusterStateIsStale(CloudSolrClientTest.java:844)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11865 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.SystemLogListenerTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/init-core-data-001
   [junit4]   2> 1192547 WARN  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 1192547 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1192548 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1192553 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1192554 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001
   [junit4]   2> 1192559 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1192559 INFO  (Thread-1893) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1192559 INFO  (Thread-1893) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1192665 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:36415
   [junit4]   2> 1192675 ERROR (Thread-1893) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1194144 INFO  (jetty-launcher-222-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1194183 INFO  (jetty-launcher-222-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1194203 INFO  (jetty-launcher-222-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1195093 INFO  (jetty-launcher-222-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@361af66d{/solr,null,AVAILABLE}
   [junit4]   2> 1195094 INFO  (jetty-launcher-222-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@c974a21{SSL,[ssl, http/1.1]}{127.0.0.1:41796}
   [junit4]   2> 1195094 INFO  (jetty-launcher-222-thread-1) [    ] o.e.j.s.Server Started @1217853ms
   [junit4]   2> 1195094 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41796}
   [junit4]   2> 1195095 ERROR (jetty-launcher-222-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1195095 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1195095 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1195095 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1195095 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-11T11:45:06.791Z
   [junit4]   2> 1195548 INFO  (jetty-launcher-222-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7e90bec4{/solr,null,AVAILABLE}
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@610f628e{SSL,[ssl, http/1.1]}{127.0.0.1:41713}
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.e.j.s.Server Started @1218307ms
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41713}
   [junit4]   2> 1195549 ERROR (jetty-launcher-222-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1195549 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-11T11:45:07.245Z
   [junit4]   2> 1195564 INFO  (jetty-launcher-222-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f8f7b24{/solr,null,AVAILABLE}
   [junit4]   2> 1195564 INFO  (jetty-launcher-222-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4122341{SSL,[ssl, http/1.1]}{127.0.0.1:50113}
   [junit4]   2> 1195564 INFO  (jetty-launcher-222-thread-3) [    ] o.e.j.s.Server Started @1218323ms
   [junit4]   2> 1195565 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50113}
   [junit4]   2> 1195565 ERROR (jetty-launcher-222-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1195565 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1195565 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1195565 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1195565 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-11T11:45:07.261Z
   [junit4]   2> 1195698 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1195739 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1195919 INFO  (jetty-launcher-222-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36415/solr
   [junit4]   2> 1195996 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1196023 INFO  (jetty-launcher-222-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36415/solr
   [junit4]   2> 1196187 INFO  (jetty-launcher-222-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36415/solr
   [junit4]   2> 1197223 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1197227 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50113_solr
   [junit4]   2> 1197238 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.c.Overseer Overseer (id=98985620554121225-127.0.0.1:50113_solr-n_0000000000) starting
   [junit4]   2> 1197286 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1197471 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41713_solr
   [junit4]   2> 1197984 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50113_solr
   [junit4]   2> 1198024 INFO  (OverseerStateUpdate-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1198025 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1198027 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1198034 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1198034 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 1198034 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1198083 INFO  (zkCallback-238-thread-2-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1198094 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1198095 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1198444 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41796_solr
   [junit4]   2> 1198461 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1198461 DEBUG (OverseerAutoScalingTriggerThread-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1198462 INFO  (zkCallback-238-thread-1-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1198550 INFO  (zkCallback-237-thread-1-processing-n:127.0.0.1:41796_solr) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1198622 INFO  (zkCallback-239-thread-2-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1198664 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1199690 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1200089 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41796.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200113 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41796.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200113 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41796.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200126 INFO  (jetty-launcher-222-thread-1) [n:127.0.0.1:41796_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node1/.
   [junit4]   2> 1200473 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41713.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200534 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41713.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200534 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41713.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1200590 INFO  (jetty-launcher-222-thread-2) [n:127.0.0.1:41713_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node2/.
   [junit4]   2> 1200819 DEBUG (ScheduledTrigger-605-thread-2) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1201256 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50113.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1201285 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50113.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1201285 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50113.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1201318 INFO  (jetty-launcher-222-thread-3) [n:127.0.0.1:50113_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node3/.
   [junit4]   2> 1201563 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1201563 INFO  (SUITE-SystemLogListenerTest-seed#[B81B7FD6062BC52A]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36415/solr ready
   [junit4]   2> 1201767 INFO  (qtp69961355-3328) [n:127.0.0.1:50113_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&name=.system&nrtReplicas=3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1201811 INFO  (OverseerThreadFactory-607-thread-1-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.CreateCollectionCmd Create collection .system
   [junit4]   2> 1201826 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1202455 INFO  (OverseerStateUpdate-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":".system",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":".system_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:41796/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1202476 INFO  (OverseerStateUpdate-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":".system",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":".system_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:41713/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1202478 INFO  (OverseerStateUpdate-98985620554121225-127.0.0.1:50113_solr-n_0000000000) [n:127.0.0.1:50113_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":".system",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":".system_shard1_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:50113/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1202826 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1203167 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1203167 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1203209 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n4&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1203209 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1203259 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1203259 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1203439 INFO  (zkCallback-237-thread-1-processing-n:127.0.0.1:41796_solr) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203439 INFO  (zkCallback-238-thread-1-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203440 INFO  (zkCallback-239-thread-2-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203440 INFO  (zkCallback-239-thread-1-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203442 INFO  (zkCallback-237-thread-2-processing-n:127.0.0.1:41796_solr) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203443 INFO  (zkCallback-238-thread-2-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1203828 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1204258 WARN  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.Config You should not use LATEST as luceneMatchVersion property: if you use this setting, and then Solr upgrades to a newer release of Lucene, sizable changes may happen. If precise back compatibility is important then you should instead explicitly specify an actual Lucene version.
   [junit4]   2> 1204389 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1204414 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1204436 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1204481 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 1204482 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 1204506 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
   [junit4]   2> 1204512 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.IndexSchema [.system_shard1_replica_n4] Schema name=_system collection or core
   [junit4]   2> 1204535 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 1204537 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
   [junit4]   2> 1204593 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 1204603 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 1204631 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 1204631 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
   [junit4]   2> 1204633 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/.system/schema.xml to /configs/.system/schema.xml.bak
   [junit4]   2> 1204633 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n1' using configuration from collection .system, trusted=true
   [junit4]   2> 1204634 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41796.solr.core..system.shard1.replica_n1' (registry 'solr.core..system.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1204634 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1204634 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [[.system_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node1/.system_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node1/./.system_shard1_replica_n1/data/]
   [junit4]   2> 1204731 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/.system/managed-schema
   [junit4]   2> 1204732 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, the non-managed schema /configs/.system/schema.xml no longer exists.
   [junit4]   2> 1204732 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n2' using configuration from collection .system, trusted=true
   [junit4]   2> 1204732 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41713.solr.core..system.shard1.replica_n2' (registry 'solr.core..system.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1204732 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1204732 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [[.system_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node2/.system_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node2/./.system_shard1_replica_n2/data/]
   [junit4]   2> 1204751 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
   [junit4]   2> 1204751 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, the non-managed schema /configs/.system/schema.xml no longer exists.
   [junit4]   2> 1204751 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n4' using configuration from collection .system, trusted=true
   [junit4]   2> 1204752 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50113.solr.core..system.shard1.replica_n4' (registry 'solr.core..system.shard1.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a7526a
   [junit4]   2> 1204752 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1204752 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.c.SolrCore [[.system_shard1_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node3/.system_shard1_replica_n4], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_B81B7FD6062BC52A-001/tempDir-001/node3/./.system_shard1_replica_n4/data/]
   [junit4]   2> 1204834 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1205838 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1206260 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1206260 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1206261 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
   [junit4]   2> 1206261 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1206315 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3777697f[.system_shard1_replica_n1] main]
   [junit4]   2> 1206319 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
   [junit4]   2> 1206319 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
   [junit4]   2> 1206336 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
   [junit4]   2> 1207232 DEBUG (ScheduledTrigger-605-thread-1) [n:127.0.0.1:50113_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:41796_solr, 127.0.0.1:50113_solr, 127.0.0.1:41713_solr]
   [junit4]   2> 1207234 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
   [junit4]   2> 1207237 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
   [junit4]   2> 1207380 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 1207380 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 145 ms
   [junit4]   2> 1207616 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1207619 INFO  (searcherExecutor-614-thread-1-processing-n:127.0.0.1:41796_solr x:.system_shard1_replica_n1 s:shard1 c:.system r:core_node3) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@3777697f[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1207622 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583769944659591168
   [junit4]   2> 1207632 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=3 found=1 timeoutin=9999ms
   [junit4]   2> 1207743 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1207743 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1207744 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
   [junit4]   2> 1207744 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1207762 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@64c57d6c[.system_shard1_replica_n4] main]
   [junit4]   2> 1207763 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
   [junit4]   2> 1207763 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
   [junit4]   2> 1207763 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
   [junit4]   2> 1207790 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1207791 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1207812 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
   [junit4]   2> 1207829 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.IndexSchema [.system_shard1_replica_n4] Schema name=_system collection or core
   [junit4]   2> 1207844 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
   [junit4]   2> 1207844 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 32 ms
   [junit4]   2> 1207845 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1207867 INFO  (searcherExecutor-616-thread-1-processing-n:127.0.0.1:50113_solr x:.system_shard1_replica_n4 s:shard1 c:.system r:core_node6) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.c.SolrCore [.system_shard1_replica_n4] Registered new searcher Searcher@64c57d6c[.system_shard1_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1207868 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
   [junit4]   2> 1207868 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1207893 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@23df21cc[.system_shard1_replica_n2] main]
   [junit4]   2> 1207894 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
   [junit4]   2> 1207894 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
   [junit4]   2> 1207894 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
   [junit4]   2> 1207939 INFO  (qtp69961355-3330) [n:127.0.0.1:50113_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583769944991989760
   [junit4]   2> 1207941 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 1207941 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1207942 INFO  (qtp11302381-3317) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583769944995135488
   [junit4]   2> 1207960 INFO  (searcherExecutor-615-thread-1-processing-n:127.0.0.1:41713_solr x:.system_shard1_replica_n2 s:shard1 c:.system r:core_node5) [n:127.0.0.1:41713_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@23df21cc[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1207961 INFO  (zkCallback-237-thread-2-processing-n:127.0.0.1:41796_solr) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1207962 INFO  (zkCallback-237-thread-1-processing-n:127.0.0.1:41796_solr) [n:127.0.0.1:41796_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1207979 INFO  (zkCallback-239-thread-3-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1207979 INFO  (zkCallback-239-thread-2-processing-n:127.0.0.1:50113_solr) [n:127.0.0.1:50113_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1207979 INFO  (zkCallback-238-thread-2-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1207979 INFO  (zkCallback-238-thread-1-processing-n:127.0.0.1:41713_solr) [n:127.0.0.1:41713_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1208162 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1208162 INFO  (qtp1101742863-3306) [n:127.0.0.1:41796_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1208162 INFO  (qtp11

[...truncated too long message...]

1:46786_solr c:localShardsTestColl s:shard2 r:core_node9 x:localShardsTestColl_shard2_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard2.leader, tag=714505596
   [junit4]   2> 668450 INFO  (coreCloseExecutor-612-thread-2) [n:127.0.0.1:46786_solr c:foo s:shard1 r:core_node3 x:foo_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=535127703
   [junit4]   2> 668456 INFO  (coreCloseExecutor-611-thread-4) [n:127.0.0.1:54252_solr c:localShardsTestColl s:shard2 r:core_node13 x:localShardsTestColl_shard2_replica_n10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.localShardsTestColl.shard2.replica_n10, tag=404477764
   [junit4]   2> 668456 INFO  (coreCloseExecutor-611-thread-4) [n:127.0.0.1:54252_solr c:localShardsTestColl s:shard2 r:core_node13 x:localShardsTestColl_shard2_replica_n10] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@270f859f: rootName = solr_54252, domain = solr.core.localShardsTestColl.shard2.replica_n10, service url = null, agent id = null] for registry solr.core.localShardsTestColl.shard2.replica_n10 / com.codahale.metrics.MetricRegistry@310f02b8
   [junit4]   2> 668462 INFO  (coreCloseExecutor-611-thread-3) [n:127.0.0.1:54252_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.foo.shard1.replica_n2, tag=695817553
   [junit4]   2> 668462 INFO  (coreCloseExecutor-611-thread-3) [n:127.0.0.1:54252_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3824408a: rootName = solr_54252, domain = solr.core.foo.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.foo.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@4b7f1c61
   [junit4]   2> 668891 INFO  (coreCloseExecutor-611-thread-3) [n:127.0.0.1:54252_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=695817553
   [junit4]   2> 668892 INFO  (coreCloseExecutor-611-thread-2) [n:127.0.0.1:54252_solr c:overwrite s:shard1 r:core_node2 x:overwrite_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.overwrite.shard1.replica_n1, tag=230315814
   [junit4]   2> 668892 INFO  (coreCloseExecutor-611-thread-2) [n:127.0.0.1:54252_solr c:overwrite s:shard1 r:core_node2 x:overwrite_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c0241c3: rootName = solr_54252, domain = solr.core.overwrite.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.overwrite.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3696d1d7
   [junit4]   2> 668898 INFO  (coreCloseExecutor-611-thread-2) [n:127.0.0.1:54252_solr c:overwrite s:shard1 r:core_node2 x:overwrite_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.overwrite.shard1.leader, tag=230315814
   [junit4]   2> 668903 INFO  (coreCloseExecutor-611-thread-8) [n:127.0.0.1:54252_solr c:multicollection1 s:shard1 r:core_node3 x:multicollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection1.shard1.replica_n1, tag=2101615180
   [junit4]   2> 668919 INFO  (coreCloseExecutor-611-thread-8) [n:127.0.0.1:54252_solr c:multicollection1 s:shard1 r:core_node3 x:multicollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@736f9479: rootName = solr_54252, domain = solr.core.multicollection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.multicollection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@336e7620
   [junit4]   2> 668926 INFO  (coreCloseExecutor-611-thread-4) [n:127.0.0.1:54252_solr c:localShardsTestColl s:shard2 r:core_node13 x:localShardsTestColl_shard2_replica_n10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard2.leader, tag=404477764
   [junit4]   2> 668926 INFO  (coreCloseExecutor-611-thread-5) [n:127.0.0.1:54252_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n16] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard3.leader, tag=1369244601
   [junit4]   2> 668926 INFO  (coreCloseExecutor-611-thread-8) [n:127.0.0.1:54252_solr c:multicollection1 s:shard1 r:core_node3 x:multicollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard1.leader, tag=2101615180
   [junit4]   2> 668985 INFO  (jetty-closer-240-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98986035701743622-127.0.0.1:56139_solr-n_0000000000) closing
   [junit4]   2> 668985 INFO  (OverseerStateUpdate-98986035701743622-127.0.0.1:56139_solr-n_0000000000) [n:127.0.0.1:56139_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:56139_solr
   [junit4]   2> 668993 WARN  (OverseerAutoScalingTriggerThread-98986035701743622-127.0.0.1:56139_solr-n_0000000000) [n:127.0.0.1:56139_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 669013 INFO  (zkCallback-256-thread-2-processing-n:127.0.0.1:54252_solr) [n:127.0.0.1:54252_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54252_solr
   [junit4]   2> 669066 INFO  (jetty-closer-240-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@89ac06a{/solr,null,UNAVAILABLE}
   [junit4]   2> 669066 INFO  (jetty-closer-240-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39967470{/solr,null,UNAVAILABLE}
   [junit4]   2> 669067 INFO  (jetty-closer-240-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3d965fde{/solr,null,UNAVAILABLE}
   [junit4]   2> 669121 ERROR (SUITE-CloudSolrClientTest-seed#[92915A1E74F774E0]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 669121 INFO  (SUITE-CloudSolrClientTest-seed#[92915A1E74F774E0]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46594 46594
   [junit4]   2> 669482 INFO  (Thread-303) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46594 46594
   [junit4]   2> 669483 WARN  (Thread-303) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 8 /solr/clusterprops.json
   [junit4]   2> 8 /solr/aliases.json
   [junit4]   2> 3 /solr/security.json
   [junit4]   2> 3 /solr/configs/conf
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 30 /solr/collections/foo/state.json
   [junit4]   2> 30 /solr/collections/localShardsTestColl/state.json
   [junit4]   2> 18 /solr/collections/collection1/state.json
   [junit4]   2> 18 /solr/collections/2nd_collection/state.json
   [junit4]   2> 16 /solr/collections/stale_state_test_col/state.json
   [junit4]   2> 14 /solr/collections/nemesis/state.json
   [junit4]   2> 14 /solr/collections/multicollection2/state.json
   [junit4]   2> 14 /solr/collections/multicollection1/state.json
   [junit4]   2> 8 /solr/clusterstate.json
   [junit4]   2> 6 /solr/collections/overwrite/state.json
   [junit4]   2> 2 /solr/collections/foo/leader_elect/shard1/election/98986035701743622-core_node6-n_0000000000
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 8 /solr/live_nodes
   [junit4]   2> 8 /solr/collections
   [junit4]   2>
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_92915A1E74F774E0-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=FST50, a_t=PostingsFormat(name=Memory), title_s=FST50, id=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=Direct)}, docValues:{_version_=DocValuesFormat(name=Asserting), multiDefault=DocValuesFormat(name=Lucene70), title_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=414, maxMBSortInHeap=5.27631931994329, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1238668e), locale=es-BO, timezone=Etc/GMT-11
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=207669128,total=449839104
   [junit4]   2> NOTE: All tests run in this JVM: [SolrQueryTest, SolrDocumentTest, CollectionAdminRequestRequiredParamsTest, RoundEvaluatorTest, SolrExampleEmbeddedTest, AbsoluteValueEvaluatorTest, EqualToEvaluatorTest, BasicHttpSolrClientTest, UniformDistributionEvaluatorTest, TestV2Request, SolrZkClientTest, JDBCStreamTest, RecursiveEvaluatorTest, ModuloEvaluatorTest, SolrExampleStreamingBinaryTest, QueryResponseTest, LengthEvaluatorTest, ClientUtilsTest, ConcurrentUpdateSolrClientTest, TestRetryUtil, CloudSolrClientMultiConstructorTest, TestSolrProperties, TestDelegationTokenResponse, SolrExampleBinaryTest, LargeVolumeBinaryJettyTest, TestBatchUpdate, JettyWebappTest, SolrParamTest, MergeIndexesEmbeddedTest, AnlysisResponseBaseTest, TestLBHttpSolrClient, TestPolicy, CloudSolrClientBuilderTest, CloudSolrClientTest]
   [junit4] Completed [90/146 (1!)] on J1 in 118.90s, 16 tests, 1 error <<< FAILURES!

[...truncated 46454 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-Tests-master - Build # 2170 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2170/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:43419","node_name":"127.0.0.1:43419_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//clusterstate.json/28)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"c8n_1x3_lf_shard1_replica_n1",           "base_url":"http://127.0.0.1:42650",           "node_name":"127.0.0.1:42650_",           "state":"down",           "type":"NRT"},         "core_node5":{           "core":"c8n_1x3_lf_shard1_replica_n2",           "base_url":"http://127.0.0.1:43419",           "node_name":"127.0.0.1:43419_",           "state":"active",           "type":"NRT",           "leader":"true"},         "core_node6":{           "state":"down",           "base_url":"http://127.0.0.1:46239",           "core":"c8n_1x3_lf_shard1_replica_n3",           "node_name":"127.0.0.1:46239_",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"3",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:43419","node_name":"127.0.0.1:43419_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//clusterstate.json/28)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"c8n_1x3_lf_shard1_replica_n1",
          "base_url":"http://127.0.0.1:42650",
          "node_name":"127.0.0.1:42650_",
          "state":"down",
          "type":"NRT"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_n2",
          "base_url":"http://127.0.0.1:43419",
          "node_name":"127.0.0.1:43419_",
          "state":"active",
          "type":"NRT",
          "leader":"true"},
        "core_node6":{
          "state":"down",
          "base_url":"http://127.0.0.1:46239",
          "core":"c8n_1x3_lf_shard1_replica_n3",
          "node_name":"127.0.0.1:46239_",
        &nb