[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 33 - Unstable

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
8 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 33 - Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/33/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
Timeout waiting for new leader null Live Nodes: [127.0.0.1:37497_solr, 127.0.0.1:42308_solr, 127.0.0.1:46243_solr] Last available state: DocCollection(collection1//collections/collection1/state.json/14)={   "pullReplicas":"0",   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node62":{           "core":"collection1_shard1_replica_n61",           "base_url":"http://127.0.0.1:45215/solr",           "node_name":"127.0.0.1:45215_solr",           "state":"down",           "type":"NRT"},         "core_node64":{           "core":"collection1_shard1_replica_n63",           "base_url":"http://127.0.0.1:42308/solr",           "node_name":"127.0.0.1:42308_solr",           "state":"active",           "type":"NRT"},         "core_node66":{           "core":"collection1_shard1_replica_n65",           "base_url":"http://127.0.0.1:46243/solr",           "node_name":"127.0.0.1:46243_solr",           "state":"active",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"3",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timeout waiting for new leader
null
Live Nodes: [127.0.0.1:37497_solr, 127.0.0.1:42308_solr, 127.0.0.1:46243_solr]
Last available state: DocCollection(collection1//collections/collection1/state.json/14)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node62":{
          "core":"collection1_shard1_replica_n61",
          "base_url":"http://127.0.0.1:45215/solr",
          "node_name":"127.0.0.1:45215_solr",
          "state":"down",
          "type":"NRT"},
        "core_node64":{
          "core":"collection1_shard1_replica_n63",
          "base_url":"http://127.0.0.1:42308/solr",
          "node_name":"127.0.0.1:42308_solr",
          "state":"active",
          "type":"NRT"},
        "core_node66":{
          "core":"collection1_shard1_replica_n65",
          "base_url":"http://127.0.0.1:46243/solr",
          "node_name":"127.0.0.1:46243_solr",
          "state":"active",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
        at __randomizedtesting.SeedInfo.seed([440889A43B6BD87F:EC14951EF92BEC55]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
        at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection(LeaderVoteWaitTimeoutTest.java:191)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.basicTest

Error Message:


Stack Trace:
java.lang.NullPointerException
        at __randomizedtesting.SeedInfo.seed([440889A43B6BD87F:B6FC9EC67FCED54C]:0)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:347)
        at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.basicTest(LeaderVoteWaitTimeoutTest.java:97)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestTlogReplica

Error Message:
ObjectTracker found 6 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, InternalHttpClient, SolrCore, MockDirectoryWrapper, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:530)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)  at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:298)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:248)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:290)  at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1190)  at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:106)  at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1163)  at org.apache.solr.cloud.ZkController.register(ZkController.java:1136)  at org.apache.solr.cloud.ZkController.register(ZkController.java:1019)  at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:187)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:214)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:899)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1061)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:530)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)  at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:530)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)  at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:499)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:949)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:530)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)  at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:762)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)  at org.eclipse.jetty.server.Server.handle(Server.java:530)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)  at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)  at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)  at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 6 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, InternalHttpClient, SolrCore, MockDirectoryWrapper, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
        at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
        at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:298)
        at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:248)
        at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:290)
        at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1190)
        at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:106)
        at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1163)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1136)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1019)
        at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:187)
        at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:214)
        at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:899)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1061)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:499)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:949)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:762)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:961)
        at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:90)
        at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:356)
        at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
        at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:455)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([440889A43B6BD87F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor91.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13191 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderVoteWaitTimeoutTest
   [junit4]   2> 2127695 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/init-core-data-001
   [junit4]   2> 2127696 WARN  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=28 numCloses=28
   [junit4]   2> 2127696 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2127747 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 2127748 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/tempDir-001
   [junit4]   2> 2127748 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2127786 INFO  (Thread-1935) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2127786 INFO  (Thread-1935) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2127802 ERROR (Thread-1935) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2127892 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:35363
   [junit4]   2> 2128196 INFO  (zkConnectionManagerCallback-8706-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128373 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2128373 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2128374 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2128374 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2128374 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 2128374 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@69875ec5{/solr,null,AVAILABLE}
   [junit4]   2> 2128375 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@236f0cd5{HTTP/1.1,[http/1.1]}{127.0.0.1:40244}
   [junit4]   2> 2128384 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2128391 INFO  (jetty-launcher-8703-thread-4) [    ] o.e.j.s.Server Started @2128541ms
   [junit4]   2> 2128391 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40244}
   [junit4]   2> 2128392 ERROR (jetty-launcher-8703-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2128392 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2128392 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2128392 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2128392 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2128392 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T02:10:32.714Z
   [junit4]   2> 2128401 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2128436 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2128436 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2128437 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 2128437 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5a18ddf9{/solr,null,AVAILABLE}
   [junit4]   2> 2128437 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5a5bed10{HTTP/1.1,[http/1.1]}{127.0.0.1:42773}
   [junit4]   2> 2128437 INFO  (jetty-launcher-8703-thread-3) [    ] o.e.j.s.Server Started @2128588ms
   [junit4]   2> 2128437 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42773}
   [junit4]   2> 2128438 ERROR (jetty-launcher-8703-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2128438 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2128438 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2128438 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2128438 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2128438 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T02:10:32.760Z
   [junit4]   2> 2128439 INFO  (zkConnectionManagerCallback-8710-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128439 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2128526 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2128526 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2128526 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 2128542 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32d7ef57{/solr,null,AVAILABLE}
   [junit4]   2> 2128542 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@19db520a{HTTP/1.1,[http/1.1]}{127.0.0.1:34582}
   [junit4]   2> 2128542 INFO  (jetty-launcher-8703-thread-2) [    ] o.e.j.s.Server Started @2128693ms
   [junit4]   2> 2128542 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34582}
   [junit4]   2> 2128543 ERROR (jetty-launcher-8703-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2128543 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2128543 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2128543 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2128543 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2128543 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T02:10:32.865Z
   [junit4]   2> 2128551 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2128551 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@471b8fc6{/solr,null,AVAILABLE}
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5bc733f7{HTTP/1.1,[http/1.1]}{127.0.0.1:35360}
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.e.j.s.Server Started @2128702ms
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35360}
   [junit4]   2> 2128552 ERROR (jetty-launcher-8703-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2128552 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2128553 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T02:10:32.874Z
   [junit4]   2> 2128561 INFO  (zkConnectionManagerCallback-8708-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128562 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2128566 INFO  (zkConnectionManagerCallback-8712-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128568 INFO  (jetty-launcher-8703-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35363/solr
   [junit4]   2> 2128583 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2128623 INFO  (zkConnectionManagerCallback-8714-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128639 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2128656 INFO  (zkConnectionManagerCallback-8718-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128658 INFO  (jetty-launcher-8703-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35363/solr
   [junit4]   2> 2128719 INFO  (jetty-launcher-8703-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35363/solr
   [junit4]   2> 2128805 INFO  (zkConnectionManagerCallback-8722-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128806 INFO  (jetty-launcher-8703-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35363/solr
   [junit4]   2> 2128809 INFO  (zkConnectionManagerCallback-8730-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128811 INFO  (zkConnectionManagerCallback-8732-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128863 INFO  (zkConnectionManagerCallback-8726-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128920 INFO  (zkConnectionManagerCallback-8736-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128921 INFO  (zkConnectionManagerCallback-8734-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2128922 INFO  (zkConnectionManagerCallback-8738-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2129133 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2129134 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42773_solr
   [junit4]   2> 2129151 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.Overseer Overseer (id=72423103583748106-127.0.0.1:42773_solr-n_0000000000) starting
   [junit4]   2> 2129321 INFO  (zkConnectionManagerCallback-8751-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2129349 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35363/solr ready
   [junit4]   2> 2129352 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42773_solr
   [junit4]   2> 2129374 INFO  (zkCallback-8750-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2129377 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 2129390 INFO  (zkCallback-8733-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2129391 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 2129407 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 2129407 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 2129461 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:42773_solr]
   [junit4]   2> 2129461 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 2129462 DEBUG (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 2129462 DEBUG (ScheduledTrigger-4522-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 2129791 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2129805 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2129807 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2129807 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34582_solr
   [junit4]   2> 2129868 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2129893 INFO  (zkCallback-8750-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2129893 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2129893 INFO  (zkCallback-8733-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2129895 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2129916 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42773.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2129934 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2129934 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40244_solr
   [junit4]   2> 2129936 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2129971 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42773.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2129971 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42773.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2130009 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2130009 INFO  (jetty-launcher-8703-thread-3) [n:127.0.0.1:42773_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/tempDir-001/node2/.
   [junit4]   2> 2130035 INFO  (zkCallback-8750-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2130055 INFO  (zkCallback-8733-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2130063 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2130063 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35360_solr
   [junit4]   2> 2130315 INFO  (zkCallback-8735-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 2130316 INFO  (zkCallback-8733-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2130334 INFO  (zkCallback-8750-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2130335 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2130335 INFO  (zkCallback-8737-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 2130371 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2130469 DEBUG (ScheduledTrigger-4522-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 2131254 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40244.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131289 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40244.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131289 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40244.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131290 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/tempDir-001/node3/.
   [junit4]   2> 2131669 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35360.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131678 DEBUG (ScheduledTrigger-4522-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 2131679 INFO  (zkConnectionManagerCallback-8756-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2131681 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2131681 INFO  (jetty-launcher-8703-thread-4) [n:127.0.0.1:40244_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35363/solr ready
   [junit4]   2> 2131740 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35360.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131740 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35360.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2131759 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/tempDir-001/node4/.
   [junit4]   2> 2131856 INFO  (zkConnectionManagerCallback-8761-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2131946 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2131947 INFO  (jetty-launcher-8703-thread-1) [n:127.0.0.1:35360_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35363/solr ready
   [junit4]   2> 2132094 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34582.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2132144 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34582.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2132144 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34582.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a4ec7f
   [junit4]   2> 2132157 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_440889A43B6BD87F-001/tempDir-001/node1/.
   [junit4]   2> 2132175 INFO  (zkConnectionManagerCallback-8766-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2132176 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2132189 INFO  (jetty-launcher-8703-thread-2) [n:127.0.0.1:34582_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35363/solr ready
   [junit4]   2> 2132392 INFO  (zkConnectionManagerCallback-8768-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2132398 INFO  (zkConnectionManagerCallback-8773-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2132399 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2132399 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35363/solr ready
   [junit4]   2> 2132429 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5a5bed10{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2132450 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1567694511
   [junit4]   2> 2132450 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2132450 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6b689f9b: rootName = solr_42773, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@65d06bae
   [junit4]   2> 2132468 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2132468 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@613853b9: rootName = solr_42773, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@523b5614
   [junit4]   2> 2132487 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2132487 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@27ba5b99: rootName = solr_42773, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4415e7a5
   [junit4]   2> 2132502 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:42773_solr
   [junit4]   2> 2132503 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2132503 INFO  (zkCallback-8733-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132525 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132525 INFO  (zkCallback-8737-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132525 INFO  (zkCallback-8760-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132525 INFO  (zkCallback-8735-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132525 INFO  (zkCallback-8750-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132538 INFO  (zkCallback-8755-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132545 INFO  (zkCallback-8772-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132682 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2132682 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.Overseer Overseer (id=72423103583748106-127.0.0.1:42773_solr-n_0000000000) closing
   [junit4]   2> 2132683 INFO  (OverseerStateUpdate-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:42773_solr
   [junit4]   2> 2132688 INFO  (zkCallback-8765-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 2132689 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 2132689 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 2132689 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 2132689 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 2132689 WARN  (OverseerAutoScalingTriggerThread-72423103583748106-127.0.0.1:42773_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2132704 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 2132704 DEBUG (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 2132719 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34582_solr
   [junit4]   2> 2132720 INFO  (zkCallback-8731-thread-1) [    ] o.a.s.c.Overseer Overseer (id=72423103583748104-127.0.0.1:34582_solr-n_0000000001) starting
   [junit4]   2> 2132741 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5a18ddf9{/solr,null,UNAVAILABLE}
   [junit4]   2> 2132742 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 2132810 INFO  (SUITE-LeaderVoteWaitTimeoutTest-seed#[440889A43B6BD87F]-worker) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T06:27:37+09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2132825 DEBUG (OverseerAutoScalingTriggerThread-72423103583748104-127.0.0.1:34582_solr-n_0000000001) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 2132825 DEBUG (OverseerAutoScalingTriggerThread-72423103583748104-127.0.0.1:34582_solr-n_0000000001) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 2132825 DEBUG (OverseerAutoScalingTriggerThread-72423103583748104-127.0.0.1:34582_solr-n_0000000001) [    ] o.a.s.c.a.OverseerTriggerThread Proc

[...truncated too long message...]

-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1356057326
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1356057326
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1356057326

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null363115947
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null363115947
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null363115947

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 19ms :: artifacts dl 2ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 191 minutes 18 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 34 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/34/

1 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:52931/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:47082/solr/MoveReplicaHDFSTest_failed_coll_true]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:52931/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:47082/solr/MoveReplicaHDFSTest_failed_coll_true]
        at __randomizedtesting.SeedInfo.seed([B5BA6B803F8F7F6D:1F77B872885CAABD]:0)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:310)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:52931/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
        ... 46 more




Build Log:
[...truncated 12290 lines...]
   [junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
   [junit4]   2> 135519 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/init-core-data-001
   [junit4]   2> 135520 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 135520 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 135521 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 135523 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001
   [junit4]   2> 135523 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 135523 INFO  (Thread-325) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 135523 INFO  (Thread-325) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 135529 ERROR (Thread-325) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 135623 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:48066
   [junit4]   2> 135629 INFO  (zkConnectionManagerCallback-299-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135637 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 135638 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 135642 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 135651 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 135656 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 135656 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 135657 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 135657 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54f39a29{/solr,null,AVAILABLE}
   [junit4]   2> 135675 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 135675 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 135675 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 135676 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16ae889f{/solr,null,AVAILABLE}
   [junit4]   2> 135681 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@23d469cf{SSL,[ssl, http/1.1]}{127.0.0.1:56929}
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.Server Started @135721ms
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56929}
   [junit4]   2> 135682 ERROR (jetty-launcher-296-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 135682 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T21:38:11.970Z
   [junit4]   2> 135683 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 135683 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 135683 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 135684 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@af1c690{SSL,[ssl, http/1.1]}{127.0.0.1:52931}
   [junit4]   2> 135684 INFO  (jetty-launcher-296-thread-4) [    ] o.e.j.s.Server Started @135724ms
   [junit4]   2> 135684 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52931}
   [junit4]   2> 135684 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d00f8cd{/solr,null,AVAILABLE}
   [junit4]   2> 135685 ERROR (jetty-launcher-296-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T21:38:11.973Z
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2626ed3e{SSL,[ssl, http/1.1]}{127.0.0.1:50739}
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.e.j.s.Server Started @135724ms
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50739}
   [junit4]   2> 135685 ERROR (jetty-launcher-296-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 135685 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T21:38:11.973Z
   [junit4]   2> 135688 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 135688 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 135688 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 135690 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ef74074{/solr,null,AVAILABLE}
   [junit4]   2> 135691 INFO  (zkConnectionManagerCallback-303-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135692 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2f3528e5{SSL,[ssl, http/1.1]}{127.0.0.1:47082}
   [junit4]   2> 135692 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.Server Started @135731ms
   [junit4]   2> 135692 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47082}
   [junit4]   2> 135693 ERROR (jetty-launcher-296-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 135693 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 135693 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 135693 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 135693 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 135693 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T21:38:11.981Z
   [junit4]   2> 135694 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 135694 INFO  (zkConnectionManagerCallback-305-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135696 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 135705 INFO  (zkConnectionManagerCallback-301-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135705 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 135707 INFO  (jetty-launcher-296-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48066/solr
   [junit4]   2> 135708 INFO  (zkConnectionManagerCallback-311-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135710 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1057c646b7c0005, likely client has closed socket
   [junit4]   2> 135712 DEBUG (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$268/1868950919@7cb35a4c
   [junit4]   2> 135713 INFO  (zkConnectionManagerCallback-307-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135717 INFO  (jetty-launcher-296-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48066/solr
   [junit4]   2> 135720 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 135720 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48066/solr
   [junit4]   2> 135734 INFO  (zkConnectionManagerCallback-317-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135735 INFO  (zkConnectionManagerCallback-313-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135736 INFO  (zkConnectionManagerCallback-321-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135737 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1057c646b7c0008, likely client has closed socket
   [junit4]   2> 135753 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48066/solr
   [junit4]   2> 135761 DEBUG (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$268/1868950919@78cd3826
   [junit4]   2> 135762 INFO  (zkConnectionManagerCallback-327-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135762 DEBUG (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$268/1868950919@32325595
   [junit4]   2> 135773 DEBUG (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$268/1868950919@1c3ce5
   [junit4]   2> 135774 INFO  (zkConnectionManagerCallback-331-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135776 INFO  (zkConnectionManagerCallback-329-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135778 INFO  (zkConnectionManagerCallback-333-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135783 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 135784 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135785 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 135785 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 135785 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 135785 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 135787 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 135787 DEBUG (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73601739663343627-127.0.0.1:50739_solr-n_0000000000
   [junit4]   2> 135787 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50739_solr
   [junit4]   2> 135788 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.Overseer Overseer (id=73601739663343627-127.0.0.1:50739_solr-n_0000000000) starting
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 135791 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 135792 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 135792 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 135792 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 135792 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 135792 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 135793 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 135793 DEBUG (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73601739663343628-127.0.0.1:47082_solr-n_0000000001
   [junit4]   2> 135793 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73601739663343627-127.0.0.1:50739_solr-n_0000000000
   [junit4]   2> 135793 DEBUG (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73601739663343627-127.0.0.1:50739_solr-n_0000000000 to know if I could be the leader
   [junit4]   2> 135794 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47082_solr
   [junit4]   2> 135795 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 135795 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135795 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135795 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135808 DEBUG (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73601739663343626-127.0.0.1:56929_solr-n_0000000002
   [junit4]   2> 135809 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73601739663343628-127.0.0.1:47082_solr-n_0000000001
   [junit4]   2> 135809 DEBUG (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73601739663343628-127.0.0.1:47082_solr-n_0000000001 to know if I could be the leader
   [junit4]   2> 135809 DEBUG (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:56929_solr as DOWN
   [junit4]   2> 135810 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 135810 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56929_solr
   [junit4]   2> 135824 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 135825 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135825 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135825 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 135825 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 135825 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 135825 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 135826 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 135827 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 135827 DEBUG (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73601739663343622-127.0.0.1:52931_solr-n_0000000003
   [junit4]   2> 135828 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73601739663343626-127.0.0.1:56929_solr-n_0000000002
   [junit4]   2> 135828 DEBUG (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73601739663343626-127.0.0.1:56929_solr-n_0000000002 to know if I could be the leader
   [junit4]   2> 135828 DEBUG (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:52931_solr as DOWN
   [junit4]   2> 135835 INFO  (zkCallback-330-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 135850 INFO  (zkConnectionManagerCallback-344-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 135852 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 135853 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135853 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135853 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 135853 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 135853 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 135853 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 135853 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 135878 INFO  (zkCallback-328-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 135880 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 135891 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135891 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 135899 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 135900 DEBUG (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:50739_solr as DOWN
   [junit4]   2> 135900 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 135900 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
   [junit4]   2> 135900 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52931_solr
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 135901 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 135901 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 135902 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 135902 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 135902 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
   [junit4]   2> 135902 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 135902 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50739_solr
   [junit4]   2> 135903 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 135912 INFO  (zkCallback-332-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:56929_solr"} current state version: 0
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:56929_solr
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:52931_solr"} current state version: 0
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:52931_solr
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:50739_solr"} current state version: 0
   [junit4]   2> 135912 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:50739_solr
   [junit4]   2> 135917 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135917 INFO  (zkCallback-332-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 135924 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135924 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135924 INFO  (zkCallback-330-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 135926 INFO  (zkCallback-328-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 135926 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135937 INFO  (zkCallback-312-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 135952 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:56929_solr, 127.0.0.1:47082_solr]
   [junit4]   2> 135965 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 135965 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 135965 DEBUG (OverseerAutoScalingTriggerThread-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 135973 INFO  (zkCallback-343-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 135989 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.scheduled_maintenance
   [junit4]   2> 135989 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 135990 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 136029 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 136030 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 136189 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56929.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136208 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56929.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136208 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56929.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136217 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001/node1/.
   [junit4]   2> 136282 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52931.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136288 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52931.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136288 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52931.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136289 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001/node4/.
   [junit4]   2> 136292 INFO  (zkConnectionManagerCallback-349-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 136297 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 136297 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 136297 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 136297 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 136297 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 136297 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 136298 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:56929_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 136336 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47082.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136343 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47082.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136343 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47082.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136344 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001/node3/.
   [junit4]   2> 136349 INFO  (zkConnectionManagerCallback-354-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 136350 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 136350 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 136350 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 136350 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 136350 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 136351 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 136351 INFO  (jetty-launcher-296-thread-4) [n:127.0.0.1:52931_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 136353 INFO  (zkConnectionManagerCallback-359-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 136353 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 136354 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 136354 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 136354 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 136354 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 136354 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 136354 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:47082_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 136533 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50739.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136538 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50739.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136538 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50739.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 136539 INFO  (jetty-launcher-296-thread-3) [n:127.0.0.1:50739_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001/node2/.
   [junit4]   2> 136578 INFO  (zkConnectionManagerCallback-361-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 136582 INFO  (zkConnectionManagerCallback-366-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 136582 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 136582 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 136582 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 136583 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 136583 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 136583 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 136583 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 136613 INFO  (qtp735547780-1994) [n:127.0.0.1:50739_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 136614 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136614 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136614 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
   [junit4]   2> 136615 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
   [junit4]   2> 136615 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136615 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136615 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
   [junit4]   2> 136616 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
   [junit4]   2> 136616 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
   [junit4]   2> 136616 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
   [junit4]   2> 136616 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{"operation":"overseerstatus"}
   [junit4]   2> 136617 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
   [junit4]   2> 136617 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : overseerstatus , {"operation":"overseerstatus"}
   [junit4]   2> 136617 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
   [junit4]   2> 136617 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 136617 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 136618 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 136618 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 136618 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
   [junit4]   2> 136622 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 136623 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: data: /solr/overseer/collection-queue-work/qnr-0000000000
   [junit4]   2> 136623 DEBUG (zkCallback-328-thread-1) [    ] o.a.s.c.OverseerTaskQueue NodeDataChanged fired on path /overseer/collection-queue-work/qnr-0000000000 state SyncConnected latchEventType null
   [junit4]   2> 136623 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136623 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
   [junit4]   2> 136623 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 136623 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 136624 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 136624 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
   [junit4]   2> 136624 DEBUG (OverseerThreadFactory-638-thread-1) [    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{leader=127.0.0.1:50739_solr,overseer_queue_size=0,overseer_work_queue_size=0,overseer_collection_queue_size=2,overseer_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=4.047438477575876,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=6.7559016666666665,medianRequestTime=8.496981,75thPcRequestTime=11.584469,95thPcRequestTime=11.584469,99thPcRequestTime=11.584469,999thPcRequestTime=11.584469},downnode={requests=3,errors=0,avgRequestsPerSecond=4.23355453514661,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.024927,medianRequestTime=0.020678,75thPcRequestTime=0.035557,95thPcRequestTime=0.035557,99thPcRequestTime=0.035557,999thPcRequestTime=0.035557}},collection_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=4.048816280986662,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=4.0924993333333335,medianRequestTime=0.493327,75thPcRequestTime=11.53833,95thPcRequestTime=11.53833,99thPcRequestTime=11.53833,999thPcRequestTime=11.53833},overseerstatus={requests=0,errors=0,avgRequestsPerSecond=0.0,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.0,medianRequestTime=0.0,75thPcRequestTime=0.0,95thPcRequestTime=0.0,99thPcRequestTime=0.0,999thPcRequestTime=0.0}},overseer_queue={},overseer_internal_queue={peek={avgRequestsPerSecond=1.3889940067978284,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=1.40858,medianRequestTime=1.40858,75thPcRequestTime=1.40858,95thPcRequestTime=1.40858,99thPcRequestTime=1.40858,999thPcRequestTime=1.40858}},collection_queue={peektopn_wait2000={avgRequestsPerSecond=1.3889439720919154,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=714.777018,medianRequestTime=714.777018,75thPcRequestTime=714.777018,95thPcRequestTime=714.777018,99thPcRequestTime=714.777018,999thPcRequestTime=714.777018}}}
   [junit4]   2> 136624 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136624 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
   [junit4]   2> 136624 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136624 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 136626 INFO  (qtp735547780-1994) [n:127.0.0.1:50739_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=13
   [junit4]   2> 136991 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 137101 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 137787 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 137963 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Logging to org.apache.logging.slf4j.Log4jLogger@1204f17f via org.mortbay.log.Slf4jLog
   [junit4]   2> 137990 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 137993 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 138269 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 138307 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_34808_hdfs____78yrqh/webapp
   [junit4]   2> 138618 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 138801 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 138801 INFO  (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 138802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 138802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 138802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 138802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 138802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 138993 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 139294 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34808
   [junit4]   2> 139994 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 140300 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 140304 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 140313 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_54965_datanode____9nb0o0/webapp
   [junit4]   2> 140787 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54965
   [junit4]   2> 140802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 140802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 140802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 140802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 140802 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 140803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 140803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 140994 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 141092 WARN  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 141093 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 141103 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_42538_datanode____.e0ol4t/webapp
   [junit4]   2> 141555 INFO  (SUITE-MoveReplicaHDFSTest-seed#[B5BA6B803F8F7F6D]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42538
   [junit4]   2> 141776 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:46510) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 141779 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:46510) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 141937 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x53b554c6407509: from storage DS-41604b67-15fe-4941-882a-43f86f471f06 node DatanodeRegistration(127.0.0.1:48323, datanodeUuid=08c21e11-934c-4afa-83eb-2e3f16f290bb, infoPort=53072, infoSecurePort=0, ipcPort=55734, storageInfo=lv=-56;cid=testClusterID;nsid=861299830;c=0), blocks: 0, hasStaleStorage: true, processing time: 2 msecs
   [junit4]   2> 141938 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x53b554c6407558: from storage DS-0b06987e-0bfa-4e53-894b-34514c4886d0 node DatanodeRegistration(127.0.0.1:58441, datanodeUuid=4e5295a5-719f-4543-9cdb-60600fd445c2, infoPort=44270, infoSecurePort=0, ipcPort=35762, storageInfo=lv=-56;cid=testClusterID;nsid=861299830;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 141938 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x53b554c6407509: from storage DS-b5c1ca72-6370-4985-8fbd-7761889c292e node DatanodeRegistration(127.0.0.1:48323, datanodeUuid=08c21e11-934c-4afa-83eb-2e3f16f290bb, infoPort=53072, infoSecurePort=0, ipcPort=55734, storageInfo=lv=-56;cid=testClusterID;nsid=861299830;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 141939 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x53b554c6407558: from storage DS-94e7c341-6cdb-49a9-be22-1393315e28b4 node DatanodeRegistration(127.0.0.1:58441, datanodeUuid=4e5295a5-719f-4543-9cdb-60600fd445c2, infoPort=44270, infoSecurePort=0, ipcPort=35762, storageInfo=lv=-56;cid=testClusterID;nsid=861299830;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 141994 DEBUG (ScheduledTrigger-636-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 142722 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testFailedMove
   [junit4]   2> 142733 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 142733 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142733 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 142733 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 142733 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 142734 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 142736 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 142755 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 142755 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 142755 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 142756 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@937f87{/solr,null,AVAILABLE}
   [junit4]   2> 142756 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@198017e3{SSL,[ssl, http/1.1]}{127.0.0.1:52901}
   [junit4]   2> 142756 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.e.j.s.Server Started @142796ms
   [junit4]   2> 142756 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52901}
   [junit4]   2> 142757 ERROR (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 142757 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 142757 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 142757 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 142757 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 142757 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-09T21:38:19.045Z
   [junit4]   2> 142776 INFO  (zkConnectionManagerCallback-370-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 142777 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 142788 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48066/solr
   [junit4]   2> 142789 INFO  (zkConnectionManagerCallback-374-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 142790 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$268/1868950919@2660111
   [junit4]   2> 142795 INFO  (zkConnectionManagerCallback-376-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 142800 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 142800 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142800 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 142800 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 142800 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 142800 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 142801 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 142801 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 142802 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73601739663343637-127.0.0.1:52901_solr-n_0000000004
   [junit4]   2> 142802 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/73601739663343622-127.0.0.1:52931_solr-n_0000000003
   [junit4]   2> 142802 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/73601739663343622-127.0.0.1:52931_solr-n_0000000003 to know if I could be the leader
   [junit4]   2> 142802 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:52901_solr as DOWN
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 142803 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 142803 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 142804 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 142804 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52901_solr
   [junit4]   2> 142804 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 142804 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142804 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:52901_solr"} current state version: 0
   [junit4]   2> 142804 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:52901_solr
   [junit4]   2> 142804 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142804 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 142805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142805 INFO  (zkCallback-332-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142806 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142806 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142806 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142806 INFO  (zkCallback-328-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142806 INFO  (zkCallback-330-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142813 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142813 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142813 INFO  (zkCallback-312-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142813 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142813 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142813 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142813 INFO  (zkCallback-343-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142834 INFO  (zkCallback-365-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142834 INFO  (zkCallback-358-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142834 INFO  (zkCallback-365-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142834 INFO  (zkCallback-353-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142834 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142834 INFO  (zkCallback-375-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142835 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 142835 INFO  (zkCallback-348-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 142905 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 142906 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 142997 DEBUG (ScheduledTrigger-636-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5
   [junit4]   2> 143035 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52901.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 143072 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52901.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 143072 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52901.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6094b8e4
   [junit4]   2> 143074 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_B5BA6B803F8F7F6D-001/tempDir-001/node5/.
   [junit4]   2> 143092 INFO  (zkConnectionManagerCallback-383-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 143093 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterprops.json
   [junit4]   2> 143093 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 143096 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 143096 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 143096 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143096 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 143097 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[B5BA6B803F8F7F6D]) [n:127.0.0.1:52901_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48066/solr ready
   [junit4]   2> 143203 INFO  (zkConnectionManagerCallback-385-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 143259 INFO  (qtp1839064828-1997) [n:127.0.0.1:47082_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&autoAddReplicas=false&name=MoveReplicaHDFSTest_failed_coll_true&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 143260 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 143260 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 143260 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
   [junit4]   2> 143268 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000002
   [junit4]   2> 143269 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
   [junit4]   2> 143269 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 143269 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
   [junit4]   2> 143269 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000002, ]
   [junit4]   2> 143269 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
   [junit4]   2> 143269 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
   [junit4]   2> 143270 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"MoveReplicaHDFSTest_failed_coll_true",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 143276 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
   [junit4]   2> 143276 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 143276 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 143276 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 143276 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
   [junit4]   2> 143277 DEBUG (OverseerCollectionConfigSetProcessor-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000002],[]
   [junit4]   2> 143277 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
   [junit4]   2> 143277 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"MoveReplicaHDFSTest_failed_coll_true",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 143277 INFO  (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection MoveReplicaHDFSTest_failed_coll_true
   [junit4]   2> 143281 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.Assign Identify nodes using default
   [junit4]   2> 143281 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/MoveReplicaHDFSTest_failed_coll_true
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143283 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143284 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143285 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143285 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143285 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143285 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143285 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143292 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:MoveReplicaHDFSTest_failed_coll_true
   [junit4]   2> 143293 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
   [junit4]   2> 143294 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 143294 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 143295 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"MoveReplicaHDFSTest_failed_coll_true",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 143295 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: MoveReplicaHDFSTest_failed_coll_true
   [junit4]   2> 143295 DEBUG (OverseerStateUpdate-73601739663343627-127.0.0.1:50739_solr-n_0000000000) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/MoveReplicaHDFSTest_failed_coll_true/state.json
   [junit4]   2> 143295 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 143296 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 143301 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 143396 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection MoveReplicaHDFSTest_failed_coll_true, shardNames [shard1, shard2] , message : {
   [junit4]   2>   "name":"MoveReplicaHDFSTest_failed_coll_true",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 143398 DEBUG (OverseerThreadFactory-638-thread-2) [    ] o.a.s.c.a.c.CreateCollecti

[...truncated too long message...]

 file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1318271955
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1318271955
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1318271955

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null997738175
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null997738175
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null997738175

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 34ms :: artifacts dl 1ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 94 minutes 13 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 35 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/35/

6 tests failed.
FAILED:  org.apache.solr.cloud.AliasIntegrationTest.testModifyPropertiesCAR

Error Message:
Error from server at https://127.0.0.1:41092/solr: Can't modify non-existent alias testModifyPropertiesCAR

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41092/solr: Can't modify non-existent alias testModifyPropertiesCAR
        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA:497041864F46F8B1]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AliasIntegrationTest.testModifyPropertiesCAR(AliasIntegrationTest.java:266)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
KeeperErrorCode = Session expired for /clusterstate.json

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA:FA21A088134BED12]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
        at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
        at org.apache.solr.common.cloud.ZkStateReader.refreshLegacyClusterState(ZkStateReader.java:560)
        at org.apache.solr.common.cloud.ZkStateReader.forceUpdateCollection(ZkStateReader.java:366)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.updateMappingsFromZk(AbstractFullDistribZkTestBase.java:681)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.updateMappingsFromZk(AbstractFullDistribZkTestBase.java:676)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:471)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:341)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestPullReplicaErrorHandling.testPullReplicaDisconnectsFromZooKeeper

Error Message:
Didn't get expected doc count. Expected: 10, Found: 0

Stack Trace:
java.lang.AssertionError: Didn't get expected doc count. Expected: 10, Found: 0
        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA:6D3CBFD1985D8F8A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:246)
        at org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:251)
        at org.apache.solr.cloud.TestPullReplicaErrorHandling.testPullReplicaDisconnectsFromZooKeeper(TestPullReplicaErrorHandling.java:222)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
        at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
        at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:     1) Thread[id=9318, name=OverseerAutoScalingTriggerThread-72435289634177026-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]         at java.lang.ClassLoader.findLoadedClass0(Native Method)         at java.lang.ClassLoader.findLoadedClass(ClassLoader.java:1038)         at java.lang.ClassLoader.loadClass(ClassLoader.java:406)         at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)         at java.lang.ClassLoader.loadClass(ClassLoader.java:357)         at org.apache.logging.log4j.util.LoaderUtil.loadClass(LoaderUtil.java:164)         at org.apache.logging.log4j.core.impl.ThrowableProxy.loadClass(ThrowableProxy.java:622)         at org.apache.logging.log4j.core.impl.ThrowableProxy.toExtendedStackTrace(ThrowableProxy.java:738)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)         at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)         at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)         at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)         at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)         at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)         at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)         at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)         at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)         at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)         at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)         at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)         at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)         at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)         at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:
   1) Thread[id=9318, name=OverseerAutoScalingTriggerThread-72435289634177026-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
        at java.lang.ClassLoader.findLoadedClass0(Native Method)
        at java.lang.ClassLoader.findLoadedClass(ClassLoader.java:1038)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:406)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        at org.apache.logging.log4j.util.LoaderUtil.loadClass(LoaderUtil.java:164)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.loadClass(ThrowableProxy.java:622)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.toExtendedStackTrace(ThrowableProxy.java:738)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)
        at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)
        at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)
        at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)
        at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)
        at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
        at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
        at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
        at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)
        at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
        at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)
        at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)
        at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA]:0)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([72759F52BDB780EA:2F3D81DB727126A5]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate(TestLargeCluster.java:578)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12235 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/init-core-data-001
   [junit4]   2> 240744 WARN  (SUITE-RecoveryAfterSoftCommitTest-seed#[72759F52BDB780EA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
   [junit4]   2> 240758 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[72759F52BDB780EA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 240759 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[72759F52BDB780EA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 240760 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[72759F52BDB780EA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 240760 INFO  (SUITE-RecoveryAfterSoftCommitTest-seed#[72759F52BDB780EA]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /bp_umc/n
   [junit4]   2> 240761 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 240797 INFO  (Thread-117) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 240797 INFO  (Thread-117) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 240798 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ZkTestServer start zk server on port:46477
   [junit4]   2> 240852 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0: ZooKeeperServer not running
   [junit4]   2> 240924 ERROR (Thread-117) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 242072 INFO  (zkConnectionManagerCallback-150-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 242117 INFO  (zkConnectionManagerCallback-152-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 242233 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 242248 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 242249 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 242251 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 242270 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 242271 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 242285 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 242286 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 242321 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 242322 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 242324 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 242374 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 243571 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 243573 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 243573 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 243573 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 243574 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3a73be7e{/bp_umc/n,null,AVAILABLE}
   [junit4]   2> 243593 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6fdcbfa5{HTTP/1.1,[http/1.1]}{127.0.0.1:46602}
   [junit4]   2> 243593 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server Started @243744ms
   [junit4]   2> 243593 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/tempDir-001/control/data, replicaType=NRT, hostContext=/bp_umc/n, hostPort=34229, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/control-001/cores}
   [junit4]   2> 243594 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 243594 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 243594 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 243594 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 243594 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 243594 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-11T05:18:09.649Z
   [junit4]   2> 243677 INFO  (zkConnectionManagerCallback-154-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 243755 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 243755 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/control-001/solr.xml
   [junit4]   2> 243771 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101576646a30002, likely client has closed socket
   [junit4]   2> 243802 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 243802 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 243813 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 243856 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46477/solr
   [junit4]   2> 243988 INFO  (zkConnectionManagerCallback-158-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 244009 INFO  (zkConnectionManagerCallback-160-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 244500 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 244501 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34229_bp_umc%2Fn
   [junit4]   2> 244519 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.Overseer Overseer (id=72435165798006788-127.0.0.1:34229_bp_umc%2Fn-n_0000000000) starting
   [junit4]   2> 244607 INFO  (zkConnectionManagerCallback-167-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 244609 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46477/solr ready
   [junit4]   2> 244640 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34229_bp_umc%2Fn
   [junit4]   2> 244673 INFO  (OverseerStateUpdate-72435165798006788-127.0.0.1:34229_bp_umc%2Fn-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 244749 INFO  (zkCallback-166-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 246054 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 246091 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 246091 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 246093 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/control-001/cores
   [junit4]   2> 246332 INFO  (zkConnectionManagerCallback-172-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 246333 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 246346 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46477/solr ready
   [junit4]   2> 246347 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39354,localport=34229], receiveBufferSize:531000
   [junit4]   2> 246362 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46602,localport=42238], receiveBufferSize=530904
   [junit4]   2> 246403 INFO  (qtp1781558814-557) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34229_bp_umc%252Fn&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 246551 INFO  (OverseerThreadFactory-197-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 246740 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39360,localport=34229], receiveBufferSize:531000
   [junit4]   2> 246756 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46602,localport=42244], receiveBufferSize=530904
   [junit4]   2> 246774 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 246774 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 246944 INFO  (zkCallback-159-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 248032 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 248215 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 249140 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 249556 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 249557 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 249557 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 249557 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 249593 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.28523959980017555]
   [junit4]   2> 249596 WARN  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 249941 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 249941 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 249975 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 249975 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 250020 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1804290038, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 250021 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@618325da[control_collection_shard1_replica_n1] main]
   [junit4]   2> 250023 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 250056 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 250057 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 250145 INFO  (searcherExecutor-200-thread-1-processing-n:127.0.0.1:34229_bp_umc%2Fn x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@618325da[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 250146 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597425735382859776
   [junit4]   2> 250240 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34229/bp_umc/n/control_collection_shard1_replica_n1/
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:34229/bp_umc/n/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 250278 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 250298 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34229/bp_umc/n/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 250384 INFO  (zkCallback-159-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 250399 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 250401 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3627
   [junit4]   2> 250437 INFO  (qtp1781558814-557) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 250600 INFO  (OverseerCollectionConfigSetProcessor-72435165798006788-127.0.0.1:34229_bp_umc%2Fn-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 250611 INFO  (zkCallback-159-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 251448 INFO  (qtp1781558814-557) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:34229_bp_umc%252Fn&wt=javabin&version=2} status=0 QTime=5044
   [junit4]   2> 251557 INFO  (zkConnectionManagerCallback-177-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 251559 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 251560 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46477/solr ready
   [junit4]   2> 251560 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 251569 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39386,localport=34229], receiveBufferSize:531000
   [junit4]   2> 251570 INFO  (SocketProxy-Acceptor-34229) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46602,localport=42270], receiveBufferSize=530904
   [junit4]   2> 251606 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 251752 INFO  (OverseerThreadFactory-197-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 251821 WARN  (OverseerThreadFactory-197-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 252098 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 252098 INFO  (qtp1781558814-561) [n:127.0.0.1:34229_bp_umc%2Fn    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=491
   [junit4]   2> 252894 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001 of type TLOG
   [junit4]   2> 252895 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 252896 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 252896 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 252896 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 252909 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b2d6fd3{/bp_umc/n,null,AVAILABLE}
   [junit4]   2> 252910 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@bc509ad{HTTP/1.1,[http/1.1]}{127.0.0.1:42204}
   [junit4]   2> 252910 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server Started @253061ms
   [junit4]   2> 252910 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/bp_umc/n, hostPort=33708, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001/cores}
   [junit4]   2> 252910 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 252910 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 252910 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 252911 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 252911 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 252911 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-11T05:18:18.966Z
   [junit4]   2> 253027 INFO  (zkConnectionManagerCallback-179-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 253100 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 253100 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001/solr.xml
   [junit4]   2> 253137 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 253138 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 253139 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 253210 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46477/solr
   [junit4]   2> 253335 INFO  (zkConnectionManagerCallback-183-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 253481 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101576646a30009, likely client has closed socket
   [junit4]   2> 253555 INFO  (zkConnectionManagerCallback-185-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 253677 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 253679 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 253694 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 253694 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33708_bp_umc%2Fn
   [junit4]   2> 253713 INFO  (zkCallback-176-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 253715 INFO  (zkCallback-159-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 253744 INFO  (zkCallback-166-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 253773 INFO  (zkCallback-184-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 253790 INFO  (OverseerCollectionConfigSetProcessor-72435165798006788-127.0.0.1:34229_bp_umc%2Fn-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 254800 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 254889 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 254900 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 254901 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001/cores
   [junit4]   2> 254992 INFO  (zkConnectionManagerCallback-192-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 255045 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 255046 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46477/solr ready
   [junit4]   2> 255202 INFO  (SocketProxy-Acceptor-33708) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=58096,localport=33708], receiveBufferSize:531000
   [junit4]   2> 255232 INFO  (SocketProxy-Acceptor-33708) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=42204,localport=55998], receiveBufferSize=530904
   [junit4]   2> 255252 INFO  (qtp833897265-623) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33708_bp_umc%252Fn&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 255333 INFO  (OverseerThreadFactory-197-thread-3) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:33708_bp_umc%2Fn for creating new replica
   [junit4]   2> 255353 INFO  (SocketProxy-Acceptor-33708) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=58100,localport=33708], receiveBufferSize:531000
   [junit4]   2> 255378 INFO  (SocketProxy-Acceptor-33708) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=42204,localport=56002], receiveBufferSize=530904
   [junit4]   2> 255407 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 257034 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 257196 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema [collection1_shard1_replica_t21] Schema name=test
   [junit4]   2> 258158 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 258290 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t21' using configuration from collection collection1, trusted=true
   [junit4]   2> 258290 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t21' (registry 'solr.core.collection1.shard1.replica_t21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 258291 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 258291 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [[collection1_shard1_replica_t21] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001/cores/collection1_shard1_replica_t21], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-1-001/cores/collection1_shard1_replica_t21/data/]
   [junit4]   2> 258325 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.28523959980017555]
   [junit4]   2> 258417 WARN  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 258854 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 258854 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 258855 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 258855 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 258869 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1804290038, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 258871 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.SolrIndexSearcher Opening [Searcher@68a33fe9[collection1_shard1_replica_t21] main]
   [junit4]   2> 258889 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 258889 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 258890 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 258890 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597425744551608320
   [junit4]   2> 258942 INFO  (searcherExecutor-211-thread-1-processing-n:127.0.0.1:33708_bp_umc%2Fn x:collection1_shard1_replica_t21 c:collection1 s:shard1) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [collection1_shard1_replica_t21] Registered new searcher Searcher@68a33fe9[collection1_shard1_replica_t21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 258945 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
   [junit4]   2> 259017 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 259017 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 259017 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33708/bp_umc/n/collection1_shard1_replica_t21/
   [junit4]   2> 259018 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 259018 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy http://127.0.0.1:33708/bp_umc/n/collection1_shard1_replica_t21/ has no replicas
   [junit4]   2> 259018 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 259071 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController collection1_shard1_replica_t21 stopping background replication from leader
   [junit4]   2> 259091 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33708/bp_umc/n/collection1_shard1_replica_t21/ shard1
   [junit4]   2> 259142 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 259250 INFO  (qtp833897265-621) [n:127.0.0.1:33708_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3842
   [junit4]   2> 259308 INFO  (qtp833897265-623) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33708_bp_umc%252Fn&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=4055
   [junit4]   2> 260226 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-2-001 of type TLOG
   [junit4]   2> 260227 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 260261 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 260261 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 260262 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 260262 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@603b3641{/bp_umc/n,null,AVAILABLE}
   [junit4]   2> 260263 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3bf6ffa6{HTTP/1.1,[http/1.1]}{127.0.0.1:33495}
   [junit4]   2> 260263 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.e.j.s.Server Started @260414ms
   [junit4]   2> 260263 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/bp_umc/n, hostPort=39958, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-2-001/cores}
   [junit4]   2> 260263 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 260334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 260334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 260334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 260334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 260334 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-11T05:18:26.389Z
   [junit4]   2> 260443 INFO  (zkConnectionManagerCallback-194-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 260461 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 260461 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-2-001/solr.xml
   [junit4]   2> 260499 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 260499 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 260512 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 260533 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46477/solr
   [junit4]   2> 260632 INFO  (zkConnectionManagerCallback-198-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 260731 INFO  (zkConnectionManagerCallback-200-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 260760 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 260762 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 260777 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 260777 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39958_bp_umc%2Fn
   [junit4]   2> 260778 INFO  (zkCallback-166-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 260800 INFO  (zkCallback-159-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 260800 INFO  (zkCallback-191-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 260804 INFO  (zkCallback-184-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 260805 INFO  (zkCallback-176-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 260926 INFO  (zkCallback-199-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 261618 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 261671 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 261671 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cbe5727
   [junit4]   2> 261685 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest_72759F52BDB780EA-001/shard-2-001/cores
   [junit4]   2> 261797 INFO  (zkConnectionManagerCallback-207-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 261869 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 261871 INFO  (TEST-RecoveryAfterSoftCommitTest.test-seed#[72759F52BDB780EA]) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46477/solr ready
   [junit4]   2> 262107 INFO  (qtp833897265-619) [n:127.0.0.1:33708_bp_umc%2Fn    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39958_bp_umc%252Fn&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 262141 INFO  (OverseerThreadFactory-197-thread-4) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:39958_bp_umc%2Fn for creating new replica
   [junit4]   2> 262145 INFO  (SocketProxy-Acceptor-39958) [    ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=55004,localport=39958], receiveBufferSize:531000
   [junit4]   2> 262160 INFO  (SocketProxy-Acceptor-39958) [    ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33495,localport=36672], receiveBufferSize=530904
   [junit4]   2> 262229 INFO  (qtp1583658440-658) [n:127.0.0.1:39958_bp_umc%2Fn    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 263399 INFO  (qtp1583658440-658) [n:127.0.0.1:39958_bp_umc%2Fn c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit

[...truncated too long message...]

-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null266059703
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null266059703
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null266059703

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1131038310
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1131038310
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1131038310

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 30ms :: artifacts dl 1ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 234 minutes 56 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 36 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/36/

6 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted
        at __randomizedtesting.SeedInfo.seed([1A9C1F6766EFEF73:226976B68DA229C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertFalse(Assert.java:68)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:860)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([1A9C1F6766EFEF73:47D401EEA929493C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1198)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:38011

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:38011
        at __randomizedtesting.SeedInfo.seed([1A9C1F6766EFEF73:92C820BDC813828B]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:425)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:341)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 44 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.StressHdfsTest

Error Message:
33 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.StressHdfsTest:     1) Thread[id=8116, name=qtp1427636769-8116, state=RUNNABLE, group=TGRP-StressHdfsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=8125, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=8173, name=OverseerCollectionConfigSetProcessor-72445571835494413-127.0.0.1:38011_-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=8174, name=OverseerAutoScalingTriggerThread-72445571835494413-127.0.0.1:38011_-n_0000000002, state=WAITING, group=Overseer autoscaling triggers]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:181)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=8119, name=qtp1427636769-8119, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=8170, name=OverseerStateUpdate-72445571835494413-127.0.0.1:38011_-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=8139, name=zkConnectionManagerCallback-2939-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=8163, name=zkConnectionManagerCallback-2939-thread-1-EventThread, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    9) Thread[id=8208, name=qtp1427636769-8208, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=8115, name=qtp1427636769-8115, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=8113, name=qtp1427636769-8113, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:158)         at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:208)         at org.apache.solr.handler.admin.CollectionsHandler.sendToOCPQueue(CollectionsHandler.java:347)         at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:257)         at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:231)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)         at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)         at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)         at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)         at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)         at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)         at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)         at org.eclipse.jetty.server.Server.handle(Server.java:530)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)         at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)         at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$243/773252923.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=8136, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=8112, name=qtp1427636769-8112, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=8118, name=qtp1427636769-8118, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=8117, name=qtp1427636769-8117, state=RUNNABLE, group=TGRP-StressHdfsTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=8183, name=ScheduledTrigger-1787-thread-4, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=8120, name=Scheduler-1945791482, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=8164, name=zkConnectionManagerCallback-2932-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)   19) Thread[id=8124, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=8162, name=zkConnectionManagerCallback-2939-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)   21) Thread[id=8176, name=ScheduledTrigger-1787-thread-2, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=8175, name=ScheduledTrigger-1787-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=8177, name=ScheduledTrigger-1787-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)         at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:58)         at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:234)         at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:595)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)         at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)         at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=8134, name=Thread-1180, state=WAITING, group=TGRP-StressHdfsTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1743)   25) Thread[id=8204, name=searcherExecutor-1791-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=8140, name=Scheduler-350073100, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=8181, name=OverseerThreadFactory-1789-thread-1, state=WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=8192, name=searcherExecutor-1790-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=8206, name=qtp1427636769-8206, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=8114, name=qtp1427636769-8114-acceptor-0@ca49fb9-ServerConnector@2a008d26{HTTP/1.1,[http/1.1]}{127.0.0.1:38011}, state=RUNNABLE, group=TGRP-StressHdfsTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=8131, name=zkConnectionManagerCallback-2932-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=8165, name=zkConnectionManagerCallback-2932-thread-1-EventThread, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   33) Thread[id=8231, name=AutoscalingActionExecutor-1788-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)         at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)         at org.apache.solr.client.solrj.cloud.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:83)         at org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:80)         at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)         at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$631/1868217076.run(Unknown Source)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$119/1996707742.run(Unknown Source)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 33 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.StressHdfsTest:
   1) Thread[id=8116, name=qtp1427636769-8116, state=RUNNABLE, group=TGRP-StressHdfsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=8125, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=8173, name=OverseerCollectionConfigSetProcessor-72445571835494413-127.0.0.1:38011_-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=8174, name=OverseerAutoScalingTriggerThread-72445571835494413-127.0.0.1:38011_-n_0000000002, state=WAITING, group=Overseer autoscaling triggers]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:181)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=8119, name=qtp1427636769-8119, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=8170, name=OverseerStateUpdate-72445571835494413-127.0.0.1:38011_-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=8139, name=zkConnectionManagerCallback-2939-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=8163, name=zkConnectionManagerCallback-2939-thread-1-EventThread, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   9) Thread[id=8208, name=qtp1427636769-8208, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=8115, name=qtp1427636769-8115, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=8113, name=qtp1427636769-8113, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:158)
        at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:208)
        at org.apache.solr.handler.admin.CollectionsHandler.sendToOCPQueue(CollectionsHandler.java:347)
        at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:257)
        at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:231)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
        at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:393)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:530)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$243/773252923.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=8136, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=8112, name=qtp1427636769-8112, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=8118, name=qtp1427636769-8118, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=8117, name=qtp1427636769-8117, state=RUNNABLE, group=TGRP-StressHdfsTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=8183, name=ScheduledTrigger-1787-thread-4, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=8120, name=Scheduler-1945791482, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=8164, name=zkConnectionManagerCallback-2932-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
  19) Thread[id=8124, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=8162, name=zkConnectionManagerCallback-2939-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
  21) Thread[id=8176, name=ScheduledTrigger-1787-thread-2, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=8175, name=ScheduledTrigger-1787-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=8177, name=ScheduledTrigger-1787-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
        at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:58)
        at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:234)
        at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:595)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=8134, name=Thread-1180, state=WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1743)
  25) Thread[id=8204, name=searcherExecutor-1791-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=8140, name=Scheduler-350073100, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=8181, name=OverseerThreadFactory-1789-thread-1, state=WAITING, group=Overseer collection creation process.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=8192, name=searcherExecutor-1790-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=8206, name=qtp1427636769-8206, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  30) Thread[id=8114, name=qtp1427636769-8114-acceptor-0@ca49fb9-ServerConnector@2a008d26{HTTP/1.1,[http/1.1]}{127.0.0.1:38011}, state=RUNNABLE, group=TGRP-StressHdfsTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  31) Thread[id=8131, name=zkConnectionManagerCallback-2932-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=8165, name=zkConnectionManagerCallback-2932-thread-1-EventThread, state=WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  33) Thread[id=8231, name=AutoscalingActionExecutor-1788-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
        at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
        at org.apache.solr.client.solrj.cloud.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:83)
        at org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:80)
        at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
        at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$631/1868217076.run(Unknown Source)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$119/1996707742.run(Unknown Source)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([1A9C1F6766EFEF73]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.StressHdfsTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8120, name=Scheduler-1945791482, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=8116, name=qtp1427636769-8116, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=8164, name=zkConnectionManagerCallback-2932-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    4) Thread[id=8162, name=zkConnectionManagerCallback-2939-thread-1-SendThread(127.0.0.1:46451), state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    5) Thread[id=8176, name=ScheduledTrigger-1787-thread-2, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=8175, name=ScheduledTrigger-1787-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=8139, name=zkConnectionManagerCallback-2939-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=8177, name=ScheduledTrigger-1787-thread-3, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=8134, name=Thread-1180, state=WAITING, group=TGRP-StressHdfsTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1743)   10) Thread[id=8204, name=searcherExecutor-1791-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=8140, name=Scheduler-350073100, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=8192, name=searcherExecutor-1790-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=8206, name=qtp1427636769-8206, state=RUNNABLE, group=TGRP-StressHdfsTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=8118, name=qtp1427636769-8118, state=TIMED_WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=8131, name=zkConnectionManagerCallback-2932-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=8117, name=qtp1427636769-8117, state=RUNNABLE, group=TGRP-StressHdfsTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=8183, name=ScheduledTrigger-1787-thread-4, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=8231, name=AutoscalingActionExecutor-1788-thread-1, state=WAITING, group=TGRP-StressHdfsTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8120, name=Scheduler-1945791482, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=8116, name=qtp1427636769-8116, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=8164, name=zkConnectionManagerCallback-2932-thread-1-SendThread(127.0.0.1:46451

[...truncated too long message...]

readPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=8192, name=searcherExecutor-1790-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   13) Thread[id=8206, name=qtp1427636769-8206, state=RUNNABLE, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   14) Thread[id=8118, name=qtp1427636769-8118, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   15) Thread[id=8131, name=zkConnectionManagerCallback-2932-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   16) Thread[id=8117, name=qtp1427636769-8117, state=RUNNABLE, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]    >         at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   17) Thread[id=8183, name=ScheduledTrigger-1787-thread-4, state=WAITING, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   18) Thread[id=8231, name=AutoscalingActionExecutor-1788-thread-1, state=WAITING, group=TGRP-StressHdfsTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > at __randomizedtesting.SeedInfo.seed([1A9C1F6766EFEF73]:0)
   [junit4] Completed [139/802 (3!)] on J2 in 175.44s, 1 test, 3 errors <<< FAILURES!

[...truncated 41799 lines...]
-documentation-lint:
     [echo] checking for broken html...
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec]
     [exec] Crawl/parse...
     [exec]
     [exec] Verify...
     [echo] Checking for missing docs...
     [exec]
     [exec] build/docs/spatial3d/org/apache/lucene/spatial3d/geom/SidedPlane.html
     [exec]   missing Methods: strictlyWithin-double-double-double-
     [exec]   missing Methods: strictlyWithin-org.apache.lucene.spatial3d.geom.Vector-
     [exec]
     [exec] Missing javadocs were found!

BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/build.xml:642: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/build.xml:101: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/build.xml:142: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/build.xml:199: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/common-build.xml:2550: exec returned: 1

Total time: 182 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 37 - Still unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/37/

2 tests failed.
FAILED:  org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth

Error Message:
Error from server at https://127.0.0.1:38520/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000002

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38520/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000002
        at __randomizedtesting.SeedInfo.seed([C0827DA6B301569D:3C38A9924B21E757]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkCreateCollection(TestMiniSolrCloudClusterSSL.java:200)
        at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithCollectionCreations(TestMiniSolrCloudClusterSSL.java:188)
        at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithNodeReplacement(TestMiniSolrCloudClusterSSL.java:138)
        at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth(TestMiniSolrCloudClusterSSL.java:110)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamingTest

Error Message:
Error from server at https://127.0.0.1:46125/solr: KeeperErrorCode = Session expired for /configs/conf

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:46125/solr: KeeperErrorCode = Session expired for /configs/conf
        at __randomizedtesting.SeedInfo.seed([39FAC171726AFECE]:0)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at org.apache.solr.client.solrj.io.stream.StreamingTest.configureCluster(StreamingTest.java:102)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12939 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1264261 INFO  (SUITE-TestMiniSolrCloudClusterSSL-seed#[C0827DA6B301569D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/init-core-data-001
   [junit4]   2> 1264261 WARN  (SUITE-TestMiniSolrCloudClusterSSL-seed#[C0827DA6B301569D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 1264276 INFO  (SUITE-TestMiniSolrCloudClusterSSL-seed#[C0827DA6B301569D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1264278 INFO  (SUITE-TestMiniSolrCloudClusterSSL-seed#[C0827DA6B301569D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1264280 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSslAndNoClientAuth
   [junit4]   2> 1264280 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.TestMiniSolrCloudClusterSSL NOTE: This Test ignores the randomized SSL & clientAuth settings selected by base class
   [junit4]   2> 1264281 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001
   [junit4]   2> 1264281 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1264281 INFO  (Thread-1380) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1264281 INFO  (Thread-1380) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1264305 ERROR (Thread-1380) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1264381 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.ZkTestServer start zk server on port:34074
   [junit4]   2> 1264384 INFO  (zkConnectionManagerCallback-1254-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264390 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T14:27:37-07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1264390 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T14:27:37-07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1264390 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T14:27:37-07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d3d11f9{/solr,null,AVAILABLE}
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1264391 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 1264392 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 1264392 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d6c0f5f{/solr,null,AVAILABLE}
   [junit4]   2> 1264392 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@261813e1{/solr,null,AVAILABLE}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@39067119{SSL,[ssl, http/1.1]}{127.0.0.1:38520}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-3) [    ] o.e.j.s.Server Started @1264562ms
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38520}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@376bd302{SSL,[ssl, http/1.1]}{127.0.0.1:46651}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-1) [    ] o.e.j.s.Server Started @1264562ms
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@72dbdfa2{SSL,[ssl, http/1.1]}{127.0.0.1:42358}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-2) [    ] o.e.j.s.Server Started @1264562ms
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46651}
   [junit4]   2> 1264396 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42358}
   [junit4]   2> 1264396 ERROR (jetty-launcher-1251-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1264397 ERROR (jetty-launcher-1251-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-15T08:00:23.033Z
   [junit4]   2> 1264397 ERROR (jetty-launcher-1251-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-15T08:00:23.033Z
   [junit4]   2> 1264397 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-15T08:00:23.033Z
   [junit4]   2> 1264414 INFO  (zkConnectionManagerCallback-1256-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264415 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1264416 INFO  (zkConnectionManagerCallback-1260-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264416 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1264417 INFO  (zkConnectionManagerCallback-1259-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264417 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1264427 INFO  (jetty-launcher-1251-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34074/solr
   [junit4]   2> 1264439 INFO  (jetty-launcher-1251-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34074/solr
   [junit4]   2> 1264439 INFO  (jetty-launcher-1251-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34074/solr
   [junit4]   2> 1264441 INFO  (zkConnectionManagerCallback-1266-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264468 INFO  (zkConnectionManagerCallback-1270-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264468 INFO  (zkConnectionManagerCallback-1272-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264472 INFO  (zkConnectionManagerCallback-1274-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264474 INFO  (zkConnectionManagerCallback-1278-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264492 INFO  (zkConnectionManagerCallback-1276-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264686 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1264686 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46651_solr
   [junit4]   2> 1264711 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1264712 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.Overseer Overseer (id=72458453100396553-127.0.0.1:46651_solr-n_0000000000) starting
   [junit4]   2> 1264712 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42358_solr
   [junit4]   2> 1264729 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1264731 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1264731 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1264734 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1264748 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1264748 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38520_solr
   [junit4]   2> 1264755 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1264755 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1264818 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1264831 INFO  (zkConnectionManagerCallback-1289-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1264833 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1264834 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34074/solr ready
   [junit4]   2> 1264837 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1264837 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46651_solr
   [junit4]   2> 1264843 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1264877 INFO  (zkCallback-1288-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1264877 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1264878 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1265021 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42358.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265029 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42358.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265029 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42358.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265030 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node2/.
   [junit4]   2> 1265036 INFO  (zkConnectionManagerCallback-1294-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1265043 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38520.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265045 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1265045 INFO  (jetty-launcher-1251-thread-2) [n:127.0.0.1:42358_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34074/solr ready
   [junit4]   2> 1265060 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38520.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265068 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38520.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265072 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node3/.
   [junit4]   2> 1265081 INFO  (zkConnectionManagerCallback-1299-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1265082 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1265083 INFO  (jetty-launcher-1251-thread-3) [n:127.0.0.1:38520_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34074/solr ready
   [junit4]   2> 1265227 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46651.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265246 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46651.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265246 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46651.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1265248 INFO  (jetty-launcher-1251-thread-1) [n:127.0.0.1:46651_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node1/.
   [junit4]   2> 1265275 INFO  (zkConnectionManagerCallback-1301-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1265278 INFO  (zkConnectionManagerCallback-1304-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1265642 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1016c944606000e, likely client has closed socket
   [junit4]   2> 1265662 INFO  (zkConnectionManagerCallback-1308-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1265663 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1265664 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34074/solr ready
   [junit4]   2> 1265696 INFO  (qtp1431597995-4777) [n:127.0.0.1:38520_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&property.config=solrconfig-tlog.xml&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&name=first_collection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1265698 INFO  (OverseerThreadFactory-1428-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection first_collection
   [junit4]   2> 1265809 INFO  (OverseerStateUpdate-72458453100396553-127.0.0.1:46651_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"first_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"first_collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:38520/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1265811 INFO  (OverseerStateUpdate-72458453100396553-127.0.0.1:46651_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"first_collection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"first_collection_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:46651/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1265813 INFO  (OverseerStateUpdate-72458453100396553-127.0.0.1:46651_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"first_collection",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"first_collection_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:42358/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1266043 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node3&name=first_collection_shard1_replica_n1&action=CREATE&numShards=3&shard=shard1&wt=javabin
   [junit4]   2> 1266095 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node5&name=first_collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
   [junit4]   2> 1266115 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node6&name=first_collection_shard3_replica_n4&action=CREATE&numShards=3&shard=shard3&wt=javabin
   [junit4]   2> 1266115 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1266234 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1266234 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1266234 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1266235 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1266235 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1266235 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1267160 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1267346 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1267362 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1267425 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.s.IndexSchema [first_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1267436 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.s.IndexSchema [first_collection_shard3_replica_n4] Schema name=test
   [junit4]   2> 1267505 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.s.IndexSchema [first_collection_shard2_replica_n2] Schema name=test
   [junit4]   2> 1268183 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1268200 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1268353 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1268416 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'first_collection_shard1_replica_n1' using configuration from collection first_collection, trusted=true
   [junit4]   2> 1268417 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38520.solr.core.first_collection.shard1.replica_n1' (registry 'solr.core.first_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1268418 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1268418 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SolrCore [[first_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node3/first_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node3/./first_collection_shard1_replica_n1/data/]
   [junit4]   2> 1268432 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'first_collection_shard3_replica_n4' using configuration from collection first_collection, trusted=true
   [junit4]   2> 1268433 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42358.solr.core.first_collection.shard3.replica_n4' (registry 'solr.core.first_collection.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1268433 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1268433 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SolrCore [[first_collection_shard3_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node2/first_collection_shard3_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node2/./first_collection_shard3_replica_n4/data/]
   [junit4]   2> 1268453 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1268454 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1268479 WARN  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1268512 WARN  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1268821 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1268821 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1268822 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1268822 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1268840 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1268840 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4727f65[first_collection_shard3_replica_n4] main]
   [junit4]   2> 1268859 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1268859 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1268860 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1268861 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1268862 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1268862 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e8e0c5[first_collection_shard1_replica_n1] main]
   [junit4]   2> 1268879 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1268896 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1268896 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1268897 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1268909 INFO  (searcherExecutor-1436-thread-1-processing-n:127.0.0.1:42358_solr x:first_collection_shard3_replica_n4 c:first_collection s:shard3 r:core_node6) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SolrCore [first_collection_shard3_replica_n4] Registered new searcher Searcher@4727f65[first_collection_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1268909 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597798327302225920
   [junit4]   2> 1268928 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1268929 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1268929 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597798327323197440
   [junit4]   2> 1268945 INFO  (searcherExecutor-1435-thread-1-processing-n:127.0.0.1:38520_solr x:first_collection_shard1_replica_n1 c:first_collection s:shard1 r:core_node3) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SolrCore [first_collection_shard1_replica_n1] Registered new searcher Searcher@1e8e0c5[first_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1269282 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ZkShardTerms Successful update of terms at /collections/first_collection/terms/shard3 to Terms{values={core_node6=0}, version=0}
   [junit4]   2> 1269282 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/first_collection/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42358/solr/first_collection_shard3_replica_n4/
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SyncStrategy https://127.0.0.1:42358/solr/first_collection_shard3_replica_n4/ has no replicas
   [junit4]   2> 1269399 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1269399 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1269399 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1269399 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38520/solr/first_collection_shard1_replica_n1/
   [junit4]   2> 1269399 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1269399 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:38520/solr/first_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1269419 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42358/solr/first_collection_shard3_replica_n4/ shard3
   [junit4]   2> 1269433 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1269453 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'first_collection_shard2_replica_n2' using configuration from collection first_collection, trusted=true
   [junit4]   2> 1269453 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46651.solr.core.first_collection.shard2.replica_n2' (registry 'solr.core.first_collection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@60bfb07
   [junit4]   2> 1269454 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1269469 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SolrCore [[first_collection_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node1/first_collection_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudClusterSSL_C0827DA6B301569D-001/tempDir-001/node1/./first_collection_shard2_replica_n2/data/]
   [junit4]   2> 1269489 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269490 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269491 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1269497 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269501 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269505 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269511 WARN  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1269517 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269542 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1269544 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node6&name=first_collection_shard3_replica_n4&action=CREATE&numShards=3&shard=shard3&wt=javabin} status=0 QTime=3428
   [junit4]   2> 1269575 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38520/solr/first_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1269975 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1269975 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1269990 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269991 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269991 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1269992 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270001 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270001 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270009 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1270009 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1270010 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1270011 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5dff1620[first_collection_shard2_replica_n2] main]
   [junit4]   2> 1270026 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1270026 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
   [junit4]   2> 1270026 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1270027 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597798328474533888
   [junit4]   2> 1270110 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1270119 INFO  (searcherExecutor-1445-thread-1-processing-n:127.0.0.1:46651_solr x:first_collection_shard2_replica_n2 c:first_collection s:shard2 r:core_node5) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SolrCore [first_collection_shard2_replica_n2] Registered new searcher Searcher@5dff1620[first_collection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1270137 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/first_collection/terms/shard2 to Terms{values={core_node5=0}, version=0}
   [junit4]   2> 1270164 INFO  (qtp1431597995-4780) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node3&name=first_collection_shard1_replica_n1&action=CREATE&numShards=3&shard=shard1&wt=javabin} status=0 QTime=4124
   [junit4]   2> 1270244 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1270244 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1270245 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:46651/solr/first_collection_shard2_replica_n2/
   [junit4]   2> 1270245 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1270245 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:46651/solr/first_collection_shard2_replica_n2/ has no replicas
   [junit4]   2> 1270245 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1270318 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:46651/solr/first_collection_shard2_replica_n2/ shard2
   [junit4]   2> 1270428 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270428 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270428 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270428 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270428 INFO  (zkCallback-1275-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270428 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270484 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1270487 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&newCollection=true&collection=first_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&coreNodeName=core_node5&name=first_collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin} status=0 QTime=4392
   [junit4]   2> 1270570 INFO  (qtp1431597995-4777) [n:127.0.0.1:38520_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1270589 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270589 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270589 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270589 INFO  (zkCallback-1275-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270590 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1270590 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1271577 INFO  (qtp1431597995-4777) [n:127.0.0.1:38520_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&property.config=solrconfig-tlog.xml&collection.configName=org.apache.solr.cloud.TestMiniSolrCloudClusterSSL&name=first_collection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2} status=0 QTime=5880
   [junit4]   2> 1271593 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: first_collection failOnTimeout:true timeout (sec):330
   [junit4]   1> -
   [junit4]   1> replica:core_node3 rstate:active live:true
   [junit4]   1> replica:core_node5 rstate:active live:true
   [junit4]   1> replica:core_node6 rstate:active live:true
   [junit4]   1> no one is recoverying
   [junit4]   2> 1271595 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: first_collection
   [junit4]   2> 1271751 INFO  (qtp1431597995-4770) [n:127.0.0.1:38520_solr c:first_collection s:shard1 r:core_node3 x:first_collection_shard1_replica_n1] o.a.s.c.S.Request [first_collection_shard1_replica_n1]  webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=first_collection:6&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&shard.url=https://127.0.0.1:38520/solr/first_collection_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1523779230283&isShard=true&wt=javabin} hits=0 status=0 QTime=55
   [junit4]   2> 1271773 INFO  (qtp61873449-4768) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.S.Request [first_collection_shard3_replica_n4]  webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=first_collection:6&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&shard.url=https://127.0.0.1:42358/solr/first_collection_shard3_replica_n4/&rows=10&version=2&q=*:*&NOW=1523779230283&isShard=true&wt=javabin} hits=0 status=0 QTime=0
   [junit4]   2> 1271777 INFO  (qtp61564826-4769) [n:127.0.0.1:46651_solr c:first_collection s:shard2 r:core_node5 x:first_collection_shard2_replica_n2] o.a.s.c.S.Request [first_collection_shard2_replica_n2]  webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=first_collection:6&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&shard.url=https://127.0.0.1:46651/solr/first_collection_shard2_replica_n2/&rows=10&version=2&q=*:*&NOW=1523779230283&isShard=true&wt=javabin} hits=0 status=0 QTime=0
   [junit4]   2> 1271783 INFO  (qtp61873449-4776) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.S.Request [first_collection_shard3_replica_n4]  webapp=/solr path=/select params={q=*:*&_stateVer_=first_collection:6&wt=javabin&version=2} hits=0 status=0 QTime=136
   [junit4]   2> 1271899 INFO  (qtp61873449-4764) [n:127.0.0.1:42358_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=58
   [junit4]   2> 1271932 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:42358:">http://127.0.0.1:42358: http protocol is not supported
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:42358
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:42358:">http://127.0.0.1:42358: http protocol is not supported
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:42358
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:42358:">http://127.0.0.1:42358: http protocol is not supported
   [junit4]   2> 1271968 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:42358
   [junit4]   2> 1272178 INFO  (OverseerCollectionConfigSetProcessor-72458453100396553-127.0.0.1:46651_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1272256 INFO  (qtp61873449-4774) [n:127.0.0.1:42358_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={} status=0 QTime=0
   [junit4]   2> 1272315 INFO  (qtp1431597995-4778) [n:127.0.0.1:38520_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=25
   [junit4]   2> 1272327 INFO  (qtp1431597995-4777) [n:127.0.0.1:38520_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:38520:">http://127.0.0.1:38520: http protocol is not supported
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:38520
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:38520:">http://127.0.0.1:38520: http protocol is not supported
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:38520
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:38520:">http://127.0.0.1:38520: http protocol is not supported
   [junit4]   2> 1272365 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:38520
   [junit4]   2> 1272439 INFO  (qtp1431597995-4778) [n:127.0.0.1:38520_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={} status=0 QTime=0
   [junit4]   2> 1272464 INFO  (qtp61564826-4763) [n:127.0.0.1:46651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 1272601 INFO  (qtp61564826-4767) [n:127.0.0.1:46651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:46651:">http://127.0.0.1:46651: http protocol is not supported
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:46651
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:46651:">http://127.0.0.1:46651: http protocol is not supported
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:46651
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.conn.UnsupportedSchemeException) caught when processing request to {}-><a href="http://127.0.0.1:46651:">http://127.0.0.1:46651: http protocol is not supported
   [junit4]   2> 1272625 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:46651
   [junit4]   2> 1272840 INFO  (qtp61564826-4771) [n:127.0.0.1:46651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={} status=0 QTime=9
   [junit4]   2> 1272842 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@72dbdfa2{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1272902 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=71021809
   [junit4]   2> 1272917 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1272917 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@44f4d1aa: rootName = solr_42358, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@587dae9a
   [junit4]   2> 1272940 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1272940 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@668d5849: rootName = solr_42358, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@442df79c
   [junit4]   2> 1272956 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1272956 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5e00cdc7: rootName = solr_42358, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@3acc4b6c
   [junit4]   2> 1272975 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272975 INFO  (zkCallback-1277-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272975 INFO  (zkCallback-1275-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272975 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272977 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:42358_solr
   [junit4]   2> 1272977 INFO  (zkCallback-1273-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272977 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/first_collection/state.json] for collection [first_collection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 1272992 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1272994 INFO  (zkCallback-1288-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273027 INFO  (coreCloseExecutor-1450-thread-1) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.c.SolrCore [first_collection_shard3_replica_n4]  CLOSING SolrCore org.apache.solr.core.SolrCore@462a4e0c
   [junit4]   2> 1273011 INFO  (zkCallback-1298-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273011 INFO  (zkCallback-1275-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273011 INFO  (zkCallback-1273-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273011 INFO  (zkCallback-1307-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273011 INFO  (zkCallback-1277-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1272994 INFO  (zkCallback-1293-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 1273027 INFO  (coreCloseExecutor-1450-thread-1) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.first_collection.shard3.replica_n4, tag=462a4e0c
   [junit4]   2> 1273028 INFO  (coreCloseExecutor-1450-thread-1) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@34a42038: rootName = solr_42358, domain = solr.core.first_collection.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.first_collection.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@135336cb
   [junit4]   2> 1273052 INFO  (coreCloseExecutor-1450-thread-1) [n:127.0.0.1:42358_solr c:first_collection s:shard3 r:core_node6 x:first_collection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.first_collection.shard3.leader, tag=462a4e0c
   [junit4]   2> 1273107 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3d6c0f5f{/solr,null,UNAVAILABLE}
   [junit4]   2> 1273137 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 1273184 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T14:27:37-07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1273242 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1273242 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1273242 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 1273243 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@647d4574{/solr,null,AVAILABLE}
   [junit4]   2> 1273270 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7aab5212{SSL,[ssl, http/1.1]}{127.0.0.1:45285}
   [junit4]   2> 1273271 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.e.j.s.Server Started @1273437ms
   [junit4]   2> 1273271 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45285}
   [junit4]   2> 1273271 ERROR (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1273271 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1273279 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1273279 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1273279 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1273279 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-15T08:00:31.915Z
   [junit4]   2> 1273293 INFO  (zkConnectionManagerCallback-1310-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1273294 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1273355 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34074/solr
   [junit4]   2> 1273358 INFO  (zkConnectionManagerCallback-1314-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1273368 INFO  (zkConnectionManagerCallback-1316-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1273382 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [n:127.0.0.1:45285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1273416 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [n:127.0.0.1:45285_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1273452 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C0827DA6B301569D]) [n:127.0.0.1:45285_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1273452 INFO  (TEST-TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth-seed#[C082

[...truncated too long message...]

dApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1340633583
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1340633583
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1340633583

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1636795045
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1636795045
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1636795045

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 38ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 229 minutes 45 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 38 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/38/

3 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:51646/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:44200/solr/MoveReplicaHDFSTest_failed_coll_true]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:51646/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:44200/solr/MoveReplicaHDFSTest_failed_coll_true]
        at __randomizedtesting.SeedInfo.seed([E4F06156EA59D0B8:4E3DB2A45D8A0568]:0)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:310)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44200/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
        ... 46 more


FAILED:  org.apache.solr.cloud.TestAuthenticationFramework.testBasics

Error Message:
Error from server at http://127.0.0.1:46202/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>  </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:46202/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>

</body>
</html>

        at __randomizedtesting.SeedInfo.seed([E4F06156EA59D0B8:D928CF7AD2B78EC8]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:127)
        at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:75)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46202/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>

</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([E4F06156EA59D0B8:B9B87FDF259F76F7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1199)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12429 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 177748 INFO  (SUITE-TestTriggerIntegration-seed#[E4F06156EA59D0B8]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_E4F06156EA59D0B8-001/init-core-data-001
   [junit4]   2> 177779 INFO  (SUITE-TestTriggerIntegration-seed#[E4F06156EA59D0B8]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 177780 INFO  (SUITE-TestTriggerIntegration-seed#[E4F06156EA59D0B8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 177852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 177852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 177856 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 177857 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 177858 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 177859 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:169) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 177859 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 177860 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 177860 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 177860 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 177860 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 177860 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 177882 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 177883 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 177884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 177884 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 177954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 177962 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 177962 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 177962 DEBUG (ScheduledTrigger-301-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178009 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178014 DEBUG (simCloudManagerPool-300-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178015 DEBUG (simCloudManagerPool-300-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 178028 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 178029 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178029 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 178030 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 178036 DEBUG (simCloudManagerPool-300-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178037 DEBUG (simCloudManagerPool-300-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 178037 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 178038 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178038 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178038 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178038 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 178038 DEBUG (ScheduledTrigger-301-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 3
   [junit4]   2> 178050 DEBUG (ScheduledTrigger-301-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178050 DEBUG (ScheduledTrigger-301-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 178059 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 178059 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 178071 DEBUG (ScheduledTrigger-301-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178079 DEBUG (ScheduledTrigger-301-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 178088 DEBUG (simCloudManagerPool-300-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178088 DEBUG (simCloudManagerPool-300-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 178089 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 178089 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178089 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 178090 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10002_solr
   [junit4]   2> 178090 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178090 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 178090 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 178090 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10002_solr]
   [junit4]   2> 178090 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c493cf6411aTdx35kmmflyy7kaacq7e5m548o",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":24097211389722906,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097211389722906],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 178091 DEBUG (ScheduledTrigger-301-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_restore_trigger]
   [junit4]   2> 178164 DEBUG (AutoscalingActionExecutor-302-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c493cf6411aTdx35kmmflyy7kaacq7e5m548o",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":24097211389722906,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097211389722906],
   [junit4]   2>     "_enqueue_time_":24097211446568306,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 178165 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 178169 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 2
   [junit4]   2> ## Empty nodes: 2
   [junit4]   2> ## Dead nodes: 1
   [junit4]   2> ## 127.0.0.1:10002_solr
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        3
   [junit4]   2> ## - update             1
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: node_lost_restore_trigger
   [junit4]   2> ## - STARTED         1
   [junit4]   2>
   [junit4]   2> 178176 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testListeners
   [junit4]   2> 178176 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 178177 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 178177 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 178178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 178180 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10002_solr
   [junit4]   2> 178180 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 178180 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:181) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 178181 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 178181 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 178181 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 178181 WARN  (AutoscalingActionExecutor-302-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_lost_restore_trigger has been closed.
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:545) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:318) ~[java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 178182 DEBUG (AutoscalingActionExecutor-302-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 975 ms for event id=559c493cf6411aTdx35kmmflyy7kaacq7e5m548o
   [junit4]   2> 178182 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 178182 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 178182 DEBUG (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 178202 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 178205 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 178206 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
   [junit4]   2> 178206 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 178214 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 178226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 178234 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178255 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178275 DEBUG (ScheduledTrigger-304-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178296 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178308 DEBUG (simCloudManagerPool-303-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178309 DEBUG (simCloudManagerPool-303-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 178313 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 178313 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178313 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 178316 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178327 DEBUG (simCloudManagerPool-303-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178327 DEBUG (simCloudManagerPool-303-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 178328 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 178328 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178328 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 178328 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}, {name=test1, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestDummyAction}], enabled=true}
   [junit4]   2> 178329 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 178331 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178331 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178336 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178351 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178352 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178355 DEBUG (simCloudManagerPool-303-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178356 DEBUG (simCloudManagerPool-303-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 178356 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 178357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 178357 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178371 DEBUG (simCloudManagerPool-303-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178371 DEBUG (simCloudManagerPool-303-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 178372 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 178372 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 178374 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178374 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178374 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 24097225615470606
   [junit4]   2> 178377 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 178394 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178395 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178395 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [24097225615470606], now=24097226642203256
   [junit4]   2> 178395 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c4c8ce1f00eTdx35kmmflyy7kaacq7e5m548q",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097225615470606,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097225615470606],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 178395 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
   [junit4]   2> 178407 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
   [junit4]   2> 178407 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c4c8ce1f00eTdx35kmmflyy7kaacq7e5m548q",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097225615470606,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097225615470606],
   [junit4]   2>     "_enqueue_time_":24097226671469306,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 178426 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 178426 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 178426 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 967 ms for event id=559c4c8ce1f00eTdx35kmmflyy7kaacq7e5m548q
   [junit4]   2> 178547 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 178562 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178562 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 178562 DEBUG (ScheduledTrigger-304-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 24097235027684506
   [junit4]   2> 178568 DEBUG (ScheduledTrigger-304-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 178583 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178583 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 178583 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [24097235027684506], now=24097236052861806
   [junit4]   2> 178583 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c4ebde4ec9aTdx35kmmflyy7kaacq7e5m548t",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097235027684506,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097235027684506],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 178583 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
   [junit4]   2> 178597 DEBUG (ScheduledTrigger-304-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10004_solr
   [junit4]   2> 178597 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c4ebde4ec9aTdx35kmmflyy7kaacq7e5m548t",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097235027684506,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097235027684506],
   [junit4]   2>     "_enqueue_time_":24097236070579756,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 178598 WARN  (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> java.lang.Exception: Error executing action: test1 for trigger event: {
   [junit4]   2>   "id":"559c4ebde4ec9aTdx35kmmflyy7kaacq7e5m548t",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097235027684506,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097235027684506],
   [junit4]   2>     "_enqueue_time_":24097236070579756,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:311) ~[java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> Caused by: java.lang.RuntimeException: failure
   [junit4]   2> at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestDummyAction.process(TestTriggerIntegration.java:909) ~[test/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308) ~[java/:?]
   [junit4]   2> ... 6 more
   [junit4]   2> 178599 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 178599 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 178599 DEBUG (AutoscalingActionExecutor-305-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 108 ms for event id=559c4ebde4ec9aTdx35kmmflyy7kaacq7e5m548t
   [junit4]   2> 178639 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testListeners
   [junit4]   2> 178639 INFO  (TEST-TestTriggerIntegration.testListeners-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 4
   [junit4]   2> ## Empty nodes: 4
   [junit4]   2> ## Dead nodes: 0
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        4
   [junit4]   2> ## - update             4
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: node_added_trigger
   [junit4]   2> ## - FAILED          1
   [junit4]   2> ## - STARTED         2
   [junit4]   2> ## - SUCCEEDED       1
   [junit4]   2>
   [junit4]   2> 178643 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 178643 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
   [junit4]   2> 178643 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 178644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 178644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 178646 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 178646 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:181) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 178647 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 178658 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 178659 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
   [junit4]   2> 178661 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1
   [junit4]   2> 178661 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 178674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 178687 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178687 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 178692 DEBUG (ScheduledTrigger-307-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178713 DEBUG (ScheduledTrigger-307-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178733 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178753 DEBUG (ScheduledTrigger-307-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178763 DEBUG (simCloudManagerPool-306-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178763 DEBUG (simCloudManagerPool-306-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
   [junit4]   2> 178765 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 178766 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178766 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 178774 DEBUG (ScheduledTrigger-307-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178779 DEBUG (simCloudManagerPool-306-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178779 DEBUG (simCloudManagerPool-306-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
   [junit4]   2> 178785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 178785 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 178785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 178785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 178786 DEBUG (ScheduledTrigger-307-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178786 DEBUG (ScheduledTrigger-307-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178786 DEBUG (ScheduledTrigger-307-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 24097246219707956
   [junit4]   2> 178794 DEBUG (ScheduledTrigger-307-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 178807 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 178807 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178807 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [24097246219707956], now=24097247258067656
   [junit4]   2> 178807 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c5158fda634Tdx35kmmflyy7kaacq7e5m548w",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097246219707956,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097246219707956],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178808 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
   [junit4]   2> 178826 DEBUG (ScheduledTrigger-307-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr
   [junit4]   2> 178828 DEBUG (AutoscalingActionExecutor-308-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c5158fda634Tdx35kmmflyy7kaacq7e5m548w",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":24097246219707956,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097246219707956],
   [junit4]   2>     "_enqueue_time_":24097247336832456,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178839 DEBUG (simCloudManagerPool-306-thread-4) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178839 DEBUG (simCloudManagerPool-306-thread-4) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
   [junit4]   2> 178843 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 178843 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 178843 DEBUG (AutoscalingActionExecutor-308-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 178843 DEBUG (AutoscalingActionExecutor-308-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 178843 DEBUG (AutoscalingActionExecutor-308-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 843 ms for event id=559c5158fda634Tdx35kmmflyy7kaacq7e5m548w
   [junit4]   2> 178884 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 178884 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 3
   [junit4]   2> ## Empty nodes: 3
   [junit4]   2> ## Dead nodes: 0
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        3
   [junit4]   2> ## - update             2
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: node_added_trigger
   [junit4]   2> ## - STARTED         1
   [junit4]   2> ## - SUCCEEDED       1
   [junit4]   2>
   [junit4]   2> 178899 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 178899 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17
   [junit4]   2> 178899 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 178900 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 178900 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 178910 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 178911 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18
   [junit4]   2> 178911 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion -1
   [junit4]   2> 178911 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18
   [junit4]   2> 178918 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10005_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 178925 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178925 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 178930 DEBUG (ScheduledTrigger-310-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178950 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178971 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178991 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179010 DEBUG (simCloudManagerPool-309-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 179010 DEBUG (simCloudManagerPool-309-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 19
   [junit4]   2> 179011 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19
   [junit4]   2> 179011 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179012 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 179012 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19
   [junit4]   2> 179026 INFO  (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 179026 DEBUG (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 179026 DEBUG (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 179026 DEBUG (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20
   [junit4]   2> 179027 INFO  (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 179027 DEBUG (simCloudManagerPool-309-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 179027 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20
   [junit4]   2> 179027 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10005_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 179028 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 179028 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 179028 DEBUG (ScheduledTrigger-310-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 179028 DEBUG (ScheduledTrigger-310-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 179031 DEBUG (ScheduledTrigger-310-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179048 DEBUG (ScheduledTrigger-310-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 179049 DEBUG (ScheduledTrigger-310-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179049 DEBUG (ScheduledTrigger-310-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10006_solr at time 24097259339058756
   [junit4]   2> 179052 DEBUG (ScheduledTrigger-310-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 179069 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 179069 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179069 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10006_solr] added at times [24097259339058756], now=24097260364299106
   [junit4]   2> 179069 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 179070 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 179089 DEBUG (ScheduledTrigger-310-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10006_solr
   [junit4]   2> 179096 DEBUG (AutoscalingActionExecutor-311-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "_enqueue_time_":24097260394183806,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 179096 INFO  (AutoscalingActionExecutor-311-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "_enqueue_time_":24097260394183806,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 179097 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 179097 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 179097 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 179097 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 179097 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 179099 WARN  (AutoscalingActionExecutor-311-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed.
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:545) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:318) ~[java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 179099 DEBUG (AutoscalingActionExecutor-311-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 158 ms for event id=559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z
   [junit4]   2> 179099 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 179099 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 179099 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 179100 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20
   [junit4]   2> 179101 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 179101 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 179101 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion -1
   [junit4]   2> 179101 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20
   [junit4]   2> 179101 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 179102 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10000_solr
   [junit4]   2> 179118 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 179118 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 179122 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 179134 DEBUG (ScheduledTrigger-313-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179134 DEBUG (ScheduledTrigger-313-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
   [junit4]   2> 179134 DEBUG (ScheduledTrigger-313-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "_dequeue_time_":24097263621250906,
   [junit4]   2>     "_enqueue_time_":24097260394183806,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 179155 DEBUG (ScheduledTrigger-313-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 179270 DEBUG (AutoscalingActionExecutor-314-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "_dequeue_time_":24097263621250906,
   [junit4]   2>     "_enqueue_time_":24097260394183806,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 179270 DEBUG (ScheduledTrigger-313-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 179270 DEBUG (ScheduledTrigger-313-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 179270 INFO  (AutoscalingActionExecutor-314-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":24097259339058756,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[24097259339058756],
   [junit4]   2>     "_dequeue_time_":24097263621250906,
   [junit4]   2>     "_enqueue_time_":24097260394183806,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 179272 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 179272 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 2
   [junit4]   2> ## Empty nodes: 2
   [junit4]   2> ## Dead nodes: 1
   [junit4]   2> ## 127.0.0.1:10000_solr
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        2
   [junit4]   2> ## - update             2
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: node_added_trigger1
   [junit4]   2> ## - STARTED         2
   [junit4]   2>
   [junit4]   2> 179288 INFO  (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventFromRestoredState
   [junit4]   2> 179288 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 21
   [junit4]   2> 179288 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 21
   [junit4]   2> 179288 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr
   [junit4]   2> 179288 INFO  (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 179290 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 179290 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 179290 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) ~[?:1.8.0_152]
   [junit4]   2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) ~[?:1.8.0_152]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:169) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 179291 DEBUG (AutoscalingActionExecutor-314-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 1045 ms for event id=559c5466f70e44Tdx35kmmflyy7kaacq7e5m548z
   [junit4]   2> 179291 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 179291 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 179291 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 179292 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 179292 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 179292 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[E4F06156EA59D0B8]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 179302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 179303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 22
   [junit4]   2> 179303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 22, lastZnodeVersion -1
   [junit4]   2> 179303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 22
   [junit4]   2> 179308 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 179308 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 179308 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 22, lastZnodeVersion 22
   [junit4]   2> 179308 DEBUG (ScheduledTrigger-316-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179329 DEBUG (ScheduledTrigger-316-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179354 DEBUG (ScheduledTrigger-316-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179374 DEBUG (ScheduledTrigger-316-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179395 DEBUG (ScheduledTrigger-316-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179406 DEBUG (simCloudManagerPool-315-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 179406 DEBUG (simCloudManagerPool-315-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 23
   [junit4]   2> 179408 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 23
   [junit4]   2> 179408 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 179408 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 23, lastZnodeVersion 23
   [junit4]   2> 179415 DEBUG (ScheduledTrigger-316-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179431 DEBUG (simCloudManagerPool-315-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 179431 DEBUG (simCloudManagerPool-315-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 24
   [junit4]   2> 179435 DEBUG (ScheduledTrigger-316-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 179439 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 24
   [junit4]   2> 179439 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 179439 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 179439 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 179439 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 24, lastZnodeVersion 24
   [junit4]   2> 179439 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 179439 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179439 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 24097278883784056
   [junit4]   2> 179456 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 179460 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 179460 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179478 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 179480 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 179480 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179498 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 179501 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 179501 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 179519 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 179521 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 179521 DEBUG (ScheduledTrigger-316-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger

[...truncated too long message...]

e = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1661465081
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1661465081
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null1661465081

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1589453459
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1589453459
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null1589453459

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 45ms :: artifacts dl 1ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 120 minutes 16 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 39 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/39/

4 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete

Error Message:
Error from server at https://127.0.0.1:49346/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>  </body> </html>

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:49346/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>

</body>
</html>

        at __randomizedtesting.SeedInfo.seed([65BF10160A4CAFC9:C645BEB38DA4456C]:0)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:948)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
        at org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete(TestCollectionsAPIViaSolrCloudCluster.java:170)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:49346/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>

</body>
</html>

        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more


FAILED:  org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([65BF10160A4CAFC9:9D6185FE72207E9A]:0)
        at org.apache.lucene.index.IndexWriter$ReaderPool.noDups(IndexWriter.java:867)
        at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:857)
        at org.apache.lucene.index.IndexWriter.numDeletesToMerge(IndexWriter.java:5233)
        at org.apache.lucene.index.LogMergePolicy.sizeDocs(LogMergePolicy.java:153)
        at org.apache.lucene.index.LogDocMergePolicy.size(LogDocMergePolicy.java:44)
        at org.apache.lucene.index.LogMergePolicy.findMerges(LogMergePolicy.java:469)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getMergeCandidatesNames(SegmentsInfoRequestHandler.java:100)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getSegmentsInfo(SegmentsInfoRequestHandler.java:59)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.handleRequestBody(SegmentsInfoRequestHandler.java:48)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2508)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:890)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:68)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosData

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([65BF10160A4CAFC9:1916324DBCA41856]:0)
        at org.apache.lucene.index.IndexWriter$ReaderPool.noDups(IndexWriter.java:867)
        at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:857)
        at org.apache.lucene.index.IndexWriter.numDeletesToMerge(IndexWriter.java:5233)
        at org.apache.lucene.index.LogMergePolicy.sizeDocs(LogMergePolicy.java:153)
        at org.apache.lucene.index.LogDocMergePolicy.size(LogDocMergePolicy.java:44)
        at org.apache.lucene.index.LogMergePolicy.findMerges(LogMergePolicy.java:469)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getMergeCandidatesNames(SegmentsInfoRequestHandler.java:100)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getSegmentsInfo(SegmentsInfoRequestHandler.java:59)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.handleRequestBody(SegmentsInfoRequestHandler.java:48)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2508)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:890)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosData(SegmentsInfoRequestHandlerTest.java:75)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfos

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([65BF10160A4CAFC9:29E887A9C0407799]:0)
        at org.apache.lucene.index.IndexWriter$ReaderPool.noDups(IndexWriter.java:867)
        at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:857)
        at org.apache.lucene.index.IndexWriter.numDeletesToMerge(IndexWriter.java:5233)
        at org.apache.lucene.index.LogMergePolicy.sizeDocs(LogMergePolicy.java:153)
        at org.apache.lucene.index.LogDocMergePolicy.size(LogDocMergePolicy.java:44)
        at org.apache.lucene.index.LogMergePolicy.findMerges(LogMergePolicy.java:469)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getMergeCandidatesNames(SegmentsInfoRequestHandler.java:100)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.getSegmentsInfo(SegmentsInfoRequestHandler.java:59)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandler.handleRequestBody(SegmentsInfoRequestHandler.java:48)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2508)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:337)
        at org.apache.solr.util.TestHarness.query(TestHarness.java:319)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:890)
        at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfos(SegmentsInfoRequestHandlerTest.java:61)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12962 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster
   [junit4]   2> 782238 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[65BF10160A4CAFC9]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/init-core-data-001
   [junit4]   2> 782239 WARN  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[65BF10160A4CAFC9]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 782239 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[65BF10160A4CAFC9]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 782241 INFO  (SUITE-TestCollectionsAPIViaSolrCloudCluster-seed#[65BF10160A4CAFC9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 782244 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001
   [junit4]   2> 782244 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 782244 INFO  (Thread-1674) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 782244 INFO  (Thread-1674) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 782253 ERROR (Thread-1674) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 782344 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.c.ZkTestServer start zk server on port:42184
   [junit4]   2> 782347 INFO  (zkConnectionManagerCallback-2055-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782366 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 782368 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 782369 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 782370 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 782371 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 782372 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 782372 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 782372 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 782373 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36ce0151{/solr,null,AVAILABLE}
   [junit4]   2> 782374 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@538d74bc{SSL,[ssl, http/1.1]}{127.0.0.1:36476}
   [junit4]   2> 782374 INFO  (jetty-launcher-2052-thread-3) [    ] o.e.j.s.Server Started @782432ms
   [junit4]   2> 782374 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36476}
   [junit4]   2> 782374 ERROR (jetty-launcher-2052-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 782374 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 782375 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 782375 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 782375 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 782375 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-16T23:27:32.674Z
   [junit4]   2> 782376 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 782376 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 782376 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 782379 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c65e6ca{/solr,null,AVAILABLE}
   [junit4]   2> 782379 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@120df202{SSL,[ssl, http/1.1]}{127.0.0.1:33022}
   [junit4]   2> 782379 INFO  (jetty-launcher-2052-thread-2) [    ] o.e.j.s.Server Started @782437ms
   [junit4]   2> 782379 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33022}
   [junit4]   2> 782380 ERROR (jetty-launcher-2052-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 782380 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 782380 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 782380 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 782380 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 782380 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-16T23:27:32.679Z
   [junit4]   2> 782381 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 782381 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 782381 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 782381 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d232bfa{/solr,null,AVAILABLE}
   [junit4]   2> 782382 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@62302fcf{SSL,[ssl, http/1.1]}{127.0.0.1:44212}
   [junit4]   2> 782382 INFO  (jetty-launcher-2052-thread-1) [    ] o.e.j.s.Server Started @782440ms
   [junit4]   2> 782382 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44212}
   [junit4]   2> 782382 ERROR (jetty-launcher-2052-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 782382 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 782382 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 782383 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 782383 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 782383 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-16T23:27:32.682Z
   [junit4]   2> 782390 INFO  (zkConnectionManagerCallback-2061-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782390 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 782407 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 782407 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 782407 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 782407 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5d2c647{/solr,null,AVAILABLE}
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2365e6a3{SSL,[ssl, http/1.1]}{127.0.0.1:59834}
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.e.j.s.Server Started @782466ms
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59834}
   [junit4]   2> 782408 ERROR (jetty-launcher-2052-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 782408 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-16T23:27:32.707Z
   [junit4]   2> 782422 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 782422 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 782423 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 782423 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2147780a{/solr,null,AVAILABLE}
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@35fdfdbf{SSL,[ssl, http/1.1]}{127.0.0.1:35763}
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.e.j.s.Server Started @782481ms
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35763}
   [junit4]   2> 782424 ERROR (jetty-launcher-2052-thread-5) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 782424 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-16T23:27:32.723Z
   [junit4]   2> 782430 INFO  (zkConnectionManagerCallback-2065-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782431 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 782434 INFO  (zkConnectionManagerCallback-2063-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782435 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 782435 INFO  (zkConnectionManagerCallback-2057-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782436 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 782459 INFO  (jetty-launcher-2052-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42184/solr
   [junit4]   2> 782461 INFO  (zkConnectionManagerCallback-2069-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782463 INFO  (zkConnectionManagerCallback-2071-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782474 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x105a0d5058b0005, likely client has closed socket
   [junit4]   2> 782473 INFO  (zkConnectionManagerCallback-2059-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782475 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 782476 INFO  (jetty-launcher-2052-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42184/solr
   [junit4]   2> 782493 INFO  (jetty-launcher-2052-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42184/solr
   [junit4]   2> 782501 INFO  (jetty-launcher-2052-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42184/solr
   [junit4]   2> 782502 INFO  (jetty-launcher-2052-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42184/solr
   [junit4]   2> 782511 INFO  (zkConnectionManagerCallback-2089-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782512 INFO  (zkConnectionManagerCallback-2075-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782512 INFO  (zkConnectionManagerCallback-2087-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782514 INFO  (zkConnectionManagerCallback-2082-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782532 INFO  (zkConnectionManagerCallback-2091-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782539 INFO  (zkConnectionManagerCallback-2096-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782548 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 782549 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33022_solr
   [junit4]   2> 782550 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.Overseer Overseer (id=73641805702955020-127.0.0.1:33022_solr-n_0000000000) starting
   [junit4]   2> 782554 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 782556 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44212_solr
   [junit4]   2> 782557 INFO  (zkConnectionManagerCallback-2093-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782565 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 782587 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 782587 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 782587 INFO  (zkConnectionManagerCallback-2097-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782589 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 782597 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 782597 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59834_solr
   [junit4]   2> 782608 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 782608 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 782609 INFO  (zkCallback-2092-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 782618 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 782621 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 782634 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 782634 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35763_solr
   [junit4]   2> 782636 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 782636 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 782650 INFO  (zkConnectionManagerCallback-2108-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 782654 INFO  (zkCallback-2092-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 782655 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 782655 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 782656 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 782657 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 782659 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 782659 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36476_solr
   [junit4]   2> 782670 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 782670 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 782670 INFO  (zkCallback-2092-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 782686 INFO  (zkCallback-2095-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 782699 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 782699 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33022_solr
   [junit4]   2> 782701 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 782701 INFO  (zkCallback-2092-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 782706 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 782761 INFO  (zkCallback-2070-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 782766 INFO  (zkCallback-2107-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (5)
   [junit4]   2> 782798 INFO  (zkCallback-2070-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 782799 INFO  (zkCallback-2095-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 783045 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35763.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783080 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36476.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783093 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44212.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783175 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36476.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783178 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59834.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783181 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35763.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783181 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35763.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783182 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36476.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783183 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node5/.
   [junit4]   2> 783183 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node3/.
   [junit4]   2> 783190 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59834.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783191 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59834.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783192 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node4/.
   [junit4]   2> 783205 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44212.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783205 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44212.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783207 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node1/.
   [junit4]   2> 783252 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33022.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783263 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33022.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783263 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33022.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 783265 INFO  (jetty-launcher-2052-thread-2) [n:127.0.0.1:33022_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node2/.
   [junit4]   2> 783312 INFO  (zkConnectionManagerCallback-2122-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783312 INFO  (zkConnectionManagerCallback-2117-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783313 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 783314 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 783315 INFO  (jetty-launcher-2052-thread-3) [n:127.0.0.1:36476_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 783318 INFO  (jetty-launcher-2052-thread-5) [n:127.0.0.1:35763_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 783338 INFO  (zkConnectionManagerCallback-2126-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783339 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 783340 INFO  (jetty-launcher-2052-thread-4) [n:127.0.0.1:59834_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 783350 INFO  (zkConnectionManagerCallback-2130-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783351 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 783352 INFO  (jetty-launcher-2052-thread-1) [n:127.0.0.1:44212_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 783543 INFO  (zkConnectionManagerCallback-2132-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783547 INFO  (zkConnectionManagerCallback-2137-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 783548 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 783549 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42184/solr ready
   [junit4]   2> 783553 INFO  (TEST-TestCollectionsAPIViaSolrCloudCluster.testStopAllStartAll-seed#[65BF10160A4CAFC9]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStopAllStartAll
   [junit4]   2> 783571 INFO  (qtp144115453-7163) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=090470f0-345d-4b75-9a71-65e6005990d0&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testStopAllStartAllCollection&nrtReplicas=2&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 783574 INFO  (qtp144115453-7163) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=090470f0-345d-4b75-9a71-65e6005990d0&replicationFactor=2&collection.configName=solrCloudCollectionConfig&maxShardsPerNode=1&name=testStopAllStartAllCollection&nrtReplicas=2&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=3
   [junit4]   2> 783575 INFO  (qtp144115453-7165) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=090470f0-345d-4b75-9a71-65e6005990d0&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 783577 INFO  (OverseerThreadFactory-2251-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection testStopAllStartAllCollection
   [junit4]   2> 783579 INFO  (qtp144115453-7165) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=090470f0-345d-4b75-9a71-65e6005990d0&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=3
   [junit4]   2> 783694 INFO  (OverseerStateUpdate-73641805702955020-127.0.0.1:33022_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testStopAllStartAllCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testStopAllStartAllCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36476/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 783705 INFO  (OverseerStateUpdate-73641805702955020-127.0.0.1:33022_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testStopAllStartAllCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testStopAllStartAllCollection_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:33022/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 783707 INFO  (OverseerStateUpdate-73641805702955020-127.0.0.1:33022_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testStopAllStartAllCollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testStopAllStartAllCollection_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:44212/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 783709 INFO  (OverseerStateUpdate-73641805702955020-127.0.0.1:33022_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testStopAllStartAllCollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testStopAllStartAllCollection_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35763/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 783931 INFO  (qtp144115453-7166) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153081391950&coreNodeName=core_node5&name=testStopAllStartAllCollection_shard1_replica_n2&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=0
   [junit4]   2> 783931 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153081391950&coreNodeName=core_node5&name=testStopAllStartAllCollection_shard1_replica_n2&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
   [junit4]   2> 783948 INFO  (qtp1123417005-7154) [n:127.0.0.1:36476_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153075190825&coreNodeName=core_node3&name=testStopAllStartAllCollection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=0
   [junit4]   2> 783954 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153075190825&coreNodeName=core_node3&name=testStopAllStartAllCollection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
   [junit4]   2> 783977 INFO  (qtp406341762-7184) [n:127.0.0.1:35763_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153090700882&coreNodeName=core_node8&name=testStopAllStartAllCollection_shard2_replica_n6&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=2
   [junit4]   2> 783977 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153090700882&coreNodeName=core_node8&name=testStopAllStartAllCollection_shard2_replica_n6&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
   [junit4]   2> 783988 INFO  (qtp162707886-7174) [n:127.0.0.1:44212_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153087920742&coreNodeName=core_node7&name=testStopAllStartAllCollection_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=0
   [junit4]   2> 783988 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=testStopAllStartAllCollection&version=2&replicaType=NRT&async=090470f0-345d-4b75-9a71-65e6005990d024173153087920742&coreNodeName=core_node7&name=testStopAllStartAllCollection_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
   [junit4]   2> 783989 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 783993 INFO  (qtp144115453-7165) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CoreAdminOperation Checking request status for : 090470f0-345d-4b75-9a71-65e6005990d024173153081391950
   [junit4]   2> 783993 INFO  (qtp144115453-7165) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=090470f0-345d-4b75-9a71-65e6005990d024173153081391950&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 784115 INFO  (zkCallback-2095-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784115 INFO  (zkCallback-2094-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784115 INFO  (zkCallback-2090-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784116 INFO  (zkCallback-2070-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784120 INFO  (zkCallback-2095-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784121 INFO  (zkCallback-2070-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784121 INFO  (zkCallback-2094-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784128 INFO  (zkCallback-2090-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testStopAllStartAllCollection/state.json] for collection [testStopAllStartAllCollection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 784580 INFO  (qtp144115453-7163) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=090470f0-345d-4b75-9a71-65e6005990d0&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 784581 INFO  (qtp144115453-7163) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=090470f0-345d-4b75-9a71-65e6005990d0&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 784989 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 784989 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 785006 INFO  (qtp144115453-7166) [n:127.0.0.1:33022_solr    ] o.a.s.h.a.CoreAdminOperation Checking request status for : 090470f0-345d-4b75-9a71-65e6005990d024173153081391950
   [junit4]   2> 785006 INFO  (qtp144115453-7166) [n:127.0.0.1:33022_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=090470f0-345d-4b75-9a71-65e6005990d024173153081391950&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 785018 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 785028 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.s.IndexSchema [testStopAllStartAllCollection_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 785031 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 785031 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testStopAllStartAllCollection_shard1_replica_n2' using configuration from collection testStopAllStartAllCollection, trusted=true
   [junit4]   2> 785035 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.s.IndexSchema [testStopAllStartAllCollection_shard2_replica_n6] Schema name=minimal
   [junit4]   2> 785038 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 785038 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'testStopAllStartAllCollection_shard2_replica_n6' using configuration from collection testStopAllStartAllCollection, trusted=true
   [junit4]   2> 785039 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 785043 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35763.solr.core.testStopAllStartAllCollection.shard2.replica_n6' (registry 'solr.core.testStopAllStartAllCollection.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 785043 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 785043 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.c.SolrCore [[testStopAllStartAllCollection_shard2_replica_n6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node5/testStopAllStartAllCollection_shard2_replica_n6], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node5/./testStopAllStartAllCollection_shard2_replica_n6/data/]
   [junit4]   2> 785050 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33022.solr.core.testStopAllStartAllCollection.shard1.replica_n2' (registry 'solr.core.testStopAllStartAllCollection.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 785050 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.s.IndexSchema [testStopAllStartAllCollection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 785050 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 785050 INFO  (parallelCoreAdminExecutor-2260-thread-1-processing-n:127.0.0.1:33022_solr 090470f0-345d-4b75-9a71-65e6005990d024173153081391950 CREATE) [n:127.0.0.1:33022_solr c:testStopAllStartAllCollection s:shard1 r:core_node5 x:testStopAllStartAllCollection_shard1_replica_n2] o.a.s.c.SolrCore [[testStopAllStartAllCollection_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node2/testStopAllStartAllCollection_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node2/./testStopAllStartAllCollection_shard1_replica_n2/data/]
   [junit4]   2> 785053 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 785053 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testStopAllStartAllCollection_shard1_replica_n1' using configuration from collection testStopAllStartAllCollection, trusted=true
   [junit4]   2> 785053 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36476.solr.core.testStopAllStartAllCollection.shard1.replica_n1' (registry 'solr.core.testStopAllStartAllCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 785053 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 785053 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.SolrCore [[testStopAllStartAllCollection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node3/testStopAllStartAllCollection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node3/./testStopAllStartAllCollection_shard1_replica_n1/data/]
   [junit4]   2> 785062 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.s.IndexSchema [testStopAllStartAllCollection_shard2_replica_n4] Schema name=minimal
   [junit4]   2> 785064 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 785065 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'testStopAllStartAllCollection_shard2_replica_n4' using configuration from collection testStopAllStartAllCollection, trusted=true
   [junit4]   2> 785065 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44212.solr.core.testStopAllStartAllCollection.shard2.replica_n4' (registry 'solr.core.testStopAllStartAllCollection.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@63fb5029
   [junit4]   2> 785065 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 785065 INFO  (parallelCoreAdminExecutor-2252-thread-1-processing-n:127.0.0.1:44212_solr 090470f0-345d-4b75-9a71-65e6005990d024173153087920742 CREATE) [n:127.0.0.1:44212_solr c:testStopAllStartAllCollection s:shard2 r:core_node7 x:testStopAllStartAllCollection_shard2_replica_n4] o.a.s.c.SolrCore [[testStopAllStartAllCollection_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node1/testStopAllStartAllCollection_shard2_replica_n4], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster_65BF10160A4CAFC9-001/tempDir-001/node1/./testStopAllStartAllCollection_shard2_replica_n4/data/]
   [junit4]   2> 785174 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 785174 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 785175 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 785175 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 785178 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12a74fde[testStopAllStartAllCollection_shard1_replica_n1] main]
   [junit4]   2> 785179 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 785179 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 785179 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/solrCloudCollectionConfig
   [junit4]   2> 785180 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/solrCloudCollectionConfig
   [junit4]   2> 785182 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 785183 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 785183 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 785183 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597947254388293632
   [junit4]   2> 785191 INFO  (searcherExecutor-2264-thread-1-processing-x:testStopAllStartAllCollection_shard1_replica_n1 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 r:core_node3 CREATE n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.SolrCore [testStopAllStartAllCollection_shard1_replica_n1] Registered new searcher Searcher@12a74fde[testStopAllStartAllCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 785199 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testStopAllStartAllCollection/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 785203 INFO  (parallelCoreAdminExecutor-2253-thread-1-processing-n:127.0.0.1:36476_solr 090470f0-345d-4b75-9a71-65e6005990d024173153075190825 CREATE) [n:127.0.0.1:36476_solr c:testStopAllStartAllCollection s:shard1 r:core_node3 x:testStopAllStartAllCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 785210 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@2ffca6e6[testStopAllStartAllCollection_shard2_replica_n6] main]
   [junit4]   2> 785214 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/solrCloudCollectionConfig
   [junit4]   2> 785214 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/solrCloudCollectionConfig
   [junit4]   2> 785215 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 785222 INFO  (parallelCoreAdminExecutor-2254-thread-1-processing-n:127.0.0.1:35763_solr 090470f0-345d-4b75-9a71-65e6005990d024173153090700882 CREATE) [n:127.0.0.1:35763_solr c:testStopAllStartAllCollection s:shard2 r:core_node8 x:testStopAllStartAllCollection_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597947254429188096
   [junit4]   2> 785225 INFO  (searcherExecutor-2263-thread-1-processing-x:testStopAllStartAllCollection_shard2_replica_n6 090470f0-345d-4b75-9

[...truncated too long message...]

s :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null11915905
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null11915905
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/null11915905

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null301728277
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null301728277
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/null301728277

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 69ms :: artifacts dl 10ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 121 minutes 20 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 40 - Failure

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/40/

2 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testTrigger

Error Message:
number of ops expected:<2> but was:<1>

Stack Trace:
java.lang.AssertionError: number of ops expected:<2> but was:<1>
        at __randomizedtesting.SeedInfo.seed([7B8FBCAAC3B8A4E:6473CD4835F4F963]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testTrigger(IndexSizeTriggerTest.java:187)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.request.TestUnInvertedFieldException

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [SolrIndexSearcher, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:325)  at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2047)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2220)  at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1957)  at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:719)  at org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)  at org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1924)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1900)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)  at org.apache.solr.handler.loader.XMLLoader.processUpdate(XMLLoader.java:281)  at org.apache.solr.handler.loader.XMLLoader.load(XMLLoader.java:188)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2508)  at org.apache.solr.servlet.DirectSolrConnection.request(DirectSolrConnection.java:125)  at org.apache.solr.util.TestHarness.update(TestHarness.java:284)  at org.apache.solr.util.BaseTestHarness.checkUpdateStatus(BaseTestHarness.java:281)  at org.apache.solr.util.BaseTestHarness.validateUpdate(BaseTestHarness.java:251)  at org.apache.solr.SolrTestCaseJ4.checkUpdateU(SolrTestCaseJ4.java:863)  at org.apache.solr.SolrTestCaseJ4.assertU(SolrTestCaseJ4.java:842)  at org.apache.solr.SolrTestCaseJ4.assertU(SolrTestCaseJ4.java:836)  at org.apache.solr.request.TestUnInvertedFieldException.createIndex(TestUnInvertedFieldException.java:73)  at org.apache.solr.request.TestUnInvertedFieldException.setUp(TestUnInvertedFieldException.java:55)  at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:762)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [SolrIndexSearcher, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:325)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2047)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2220)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1957)
        at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:719)
        at org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
        at org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1924)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1900)
        at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
        at org.apache.solr.handler.loader.XMLLoader.processUpdate(XMLLoader.java:281)
        at org.apache.solr.handler.loader.XMLLoader.load(XMLLoader.java:188)
        at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2508)
        at org.apache.solr.servlet.DirectSolrConnection.request(DirectSolrConnection.java:125)
        at org.apache.solr.util.TestHarness.update(TestHarness.java:284)
        at org.apache.solr.util.BaseTestHarness.checkUpdateStatus(BaseTestHarness.java:281)
        at org.apache.solr.util.BaseTestHarness.validateUpdate(BaseTestHarness.java:251)
        at org.apache.solr.SolrTestCaseJ4.checkUpdateU(SolrTestCaseJ4.java:863)
        at org.apache.solr.SolrTestCaseJ4.assertU(SolrTestCaseJ4.java:842)
        at org.apache.solr.SolrTestCaseJ4.assertU(SolrTestCaseJ4.java:836)
        at org.apache.solr.request.TestUnInvertedFieldException.createIndex(TestUnInvertedFieldException.java:73)
        at org.apache.solr.request.TestUnInvertedFieldException.setUp(TestUnInvertedFieldException.java:55)
        at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
        at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:762)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
        at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1051)
        at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:647)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([7B8FBCAAC3B8A4E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13571 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest
   [junit4]   2> 1247995 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.IndexSizeTriggerTest_7B8FBCAAC3B8A4E-001/init-core-data-001
   [junit4]   2> 1247996 WARN  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 1247996 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1247998 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1247999 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.IndexSizeTriggerTest_7B8FBCAAC3B8A4E-001/tempDir-001
   [junit4]   2> 1247999 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1248000 INFO  (Thread-5700) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1248000 INFO  (Thread-5700) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1248005 ERROR (Thread-5700) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1248100 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:56403
   [junit4]   2> 1248129 INFO  (zkConnectionManagerCallback-3365-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248138 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1248139 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1248189 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1248189 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1248189 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 1248213 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61d5d87e{/solr,null,AVAILABLE}
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@410743e8{SSL,[ssl, http/1.1]}{127.0.0.1:55603}
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.e.j.s.Server Started @1248253ms
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55603}
   [junit4]   2> 1248214 ERROR (jetty-launcher-3362-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1248214 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-18T04:43:45.273Z
   [junit4]   2> 1248281 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1248282 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1248282 INFO  (zkConnectionManagerCallback-3367-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248282 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 1248282 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1248289 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@145b33cb{/solr,null,AVAILABLE}
   [junit4]   2> 1248289 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6683ce71{SSL,[ssl, http/1.1]}{127.0.0.1:56863}
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.e.j.s.Server Started @1248329ms
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56863}
   [junit4]   2> 1248290 ERROR (jetty-launcher-3362-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1248290 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-18T04:43:45.349Z
   [junit4]   2> 1248462 INFO  (jetty-launcher-3362-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56403/solr
   [junit4]   2> 1248477 INFO  (zkConnectionManagerCallback-3369-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248478 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1248482 INFO  (zkConnectionManagerCallback-3373-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248483 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x105a71cdd650003, likely client has closed socket
   [junit4]   2> 1248506 INFO  (jetty-launcher-3362-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56403/solr
   [junit4]   2> 1248557 INFO  (zkConnectionManagerCallback-3379-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248558 INFO  (zkConnectionManagerCallback-3375-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248568 INFO  (zkConnectionManagerCallback-3381-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248687 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1248688 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56863_solr
   [junit4]   2> 1248689 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.Overseer Overseer (id=73648711336787974-127.0.0.1:56863_solr-n_0000000000) starting
   [junit4]   2> 1248703 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1248708 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55603_solr
   [junit4]   2> 1248768 INFO  (zkCallback-3380-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248797 INFO  (zkCallback-3374-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248797 INFO  (zkConnectionManagerCallback-3390-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1248798 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1248798 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56403/solr ready
   [junit4]   2> 1248967 DEBUG (OverseerStateUpdate-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 1248980 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1248980 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56863_solr
   [junit4]   2> 1248980 INFO  (zkCallback-3374-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1248985 INFO  (zkCallback-3380-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1248997 INFO  (zkCallback-3389-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1249005 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1249006 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1249006 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 1249006 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1249007 DEBUG (OverseerStateUpdate-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:56863_solr"} current state version: 0
   [junit4]   2> 1249014 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:55603_solr, 127.0.0.1:56863_solr]
   [junit4]   2> 1249021 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1249021 DEBUG (OverseerAutoScalingTriggerThread-73648711336787974-127.0.0.1:56863_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1249059 DEBUG (ScheduledTrigger-5452-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249164 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56863.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249179 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56863.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249179 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56863.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249180 INFO  (jetty-launcher-3362-thread-2) [n:127.0.0.1:56863_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.IndexSizeTriggerTest_7B8FBCAAC3B8A4E-001/tempDir-001/node2/.
   [junit4]   2> 1249257 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55603.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249264 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55603.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249264 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55603.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3c6045d2
   [junit4]   2> 1249266 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.IndexSizeTriggerTest_7B8FBCAAC3B8A4E-001/tempDir-001/node1/.
   [junit4]   2> 1249268 INFO  (zkConnectionManagerCallback-3395-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1249269 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1249269 INFO  (jetty-launcher-3362-thread-1) [n:127.0.0.1:55603_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56403/solr ready
   [junit4]   2> 1249290 INFO  (zkConnectionManagerCallback-3397-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1249292 INFO  (zkConnectionManagerCallback-3402-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1249293 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1249293 INFO  (SUITE-IndexSizeTriggerTest-seed#[7B8FBCAAC3B8A4E]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56403/solr ready
   [junit4]   2> 1249297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1249297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 1249297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1249297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1249299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1249299 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1249300 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 1249300 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249349 DEBUG (ScheduledTrigger-5461-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249369 DEBUG (ScheduledTrigger-5461-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249390 DEBUG (ScheduledTrigger-5461-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249410 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249430 DEBUG (ScheduledTrigger-5461-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249450 DEBUG (ScheduledTrigger-5461-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249470 DEBUG (ScheduledTrigger-5461-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249491 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249511 DEBUG (ScheduledTrigger-5461-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249520 INFO  (TEST-IndexSizeTriggerTest.testMergeIntegration-seed#[7B8FBCAAC3B8A4E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMergeIntegration
   [junit4]   2> 1249531 DEBUG (ScheduledTrigger-5461-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249556 DEBUG (ScheduledTrigger-5461-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249565 DEBUG (simCloudManagerPool-5460-thread-6) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testMergeIntegration_collection / shard2: {"core_node3":{
   [junit4]   2>     "core":"testMergeIntegration_collection_shard2_replica_n3",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10005_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testMergeIntegration_collection"}}
   [junit4]   2> 1249565 DEBUG (simCloudManagerPool-5460-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testMergeIntegration_collection / shard1: {"core_node1":{
   [junit4]   2>     "core":"testMergeIntegration_collection_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10005_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testMergeIntegration_collection"}}
   [junit4]   2> 1249576 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249596 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249616 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249635 DEBUG (simCloudManagerPool-5460-thread-19) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1249635 DEBUG (simCloudManagerPool-5460-thread-19) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1249636 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1249636 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1249636 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1249638 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249655 DEBUG (simCloudManagerPool-5460-thread-20) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1249655 DEBUG (simCloudManagerPool-5460-thread-20) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1249656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1249656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1249656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1249658 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249662 DEBUG (simCloudManagerPool-5460-thread-21) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1249662 DEBUG (simCloudManagerPool-5460-thread-21) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1249662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1249662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1249662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1249678 DEBUG (ScheduledTrigger-5461-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249699 DEBUG (ScheduledTrigger-5461-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249719 DEBUG (ScheduledTrigger-5461-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249739 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1249752 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5657b6d07d6eebT55f5hrcejmbm8d6y6bt6w6cxf",
   [junit4]   2>   "source":"index_size_trigger",
   [junit4]   2>   "eventTime":24303290690662123,
   [junit4]   2>   "eventType":"INDEXSIZE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "aboveSize":{},
   [junit4]   2>     "belowSize":{"testMergeIntegration_collection":[
   [junit4]   2>         "{\"core_node1\":{\n    \"core\":\"testMergeIntegration_collection_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":7,\n    \"SEARCHER.searcher.deletedDocs\":5,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":2,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":2,\n    \"__bytes__\":1,\n    \"shard\":\"shard1\",\n    \"collection\":\"testMergeIntegration_collection\"}}",
   [junit4]   2>         "{\"core_node3\":{\n    \"core\":\"testMergeIntegration_collection_shard2_replica_n3\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":3,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":3,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":3,\n    \"__bytes__\":1,\n    \"shard\":\"shard2\",\n    \"collection\":\"testMergeIntegration_collection\"}}"]},
   [junit4]   2>     "requestedOps":["Op{action=MERGESHARDS, hints={COLL_SHARD=[{\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard2\"}, {\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 1249756 DEBUG (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [index_size_trigger, .auto_add_replicas, .scheduled_maintenance]
   [junit4]   2> 1249757 INFO  (ScheduledTrigger-5461-thread-2) [    ] o.a.s.c.a.IndexSizeTriggerTest =======> CapturedEvent{timestamp=24303296355791023, stage=STARTED, actionName='null', event={
   [junit4]   2>   "id":"5657b6d07d6eebT55f5hrcejmbm8d6y6bt6w6cxf",
   [junit4]   2>   "source":"index_size_trigger",
   [junit4]   2>   "eventTime":24303290690662123,
   [junit4]   2>   "eventType":"INDEXSIZE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "aboveSize":{},
   [junit4]   2>     "belowSize":{"testMergeIntegration_collection":[
   [junit4]   2>         "{\"core_node1\":{\n    \"core\":\"testMergeIntegration_collection_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":7,\n    \"SEARCHER.searcher.deletedDocs\":5,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":2,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":2,\n    \"__bytes__\":1,\n    \"shard\":\"shard1\",\n    \"collection\":\"testMergeIntegration_collection\"}}",
   [junit4]   2>         "{\"core_node3\":{\n    \"core\":\"testMergeIntegration_collection_shard2_replica_n3\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":3,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":3,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":3,\n    \"__bytes__\":1,\n    \"shard\":\"shard2\",\n    \"collection\":\"testMergeIntegration_collection\"}}"]},
   [junit4]   2>     "_enqueue_time_":24303296322753223,
   [junit4]   2>     "requestedOps":["Op{action=MERGESHARDS, hints={COLL_SHARD=[{\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard2\"}, {\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard1\"}]}}"]}}, context={}, config={
   [junit4]   2>   "trigger":"index_size_trigger",
   [junit4]   2>   "stage":[
   [junit4]   2>     "STARTED",
   [junit4]   2>     "ABORTED",
   [junit4]   2>     "SUCCEEDED",
   [junit4]   2>     "FAILED"],
   [junit4]   2>   "afterAction":[
   [junit4]   2>     "compute_plan",
   [junit4]   2>     "execute_plan"],
   [junit4]   2>   "class":"org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest$CapturingTriggerListener",
   [junit4]   2>   "beforeAction":[
   [junit4]   2>     "compute_plan",
   [junit4]   2>     "execute_plan"]}, message='null'}
   [junit4]   2> 1249759 DEBUG (AutoscalingActionExecutor-5462-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5657b6d07d6eebT55f5hrcejmbm8d6y6bt6w6cxf",
   [junit4]   2>   "source":"index_size_trigger",
   [junit4]   2>   "eventTime":24303290690662123,
   [junit4]   2>   "eventType":"INDEXSIZE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "aboveSize":{},
   [junit4]   2>     "belowSize":{"testMergeIntegration_collection":[
   [junit4]   2>         "{\"core_node1\":{\n    \"core\":\"testMergeIntegration_collection_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":7,\n    \"SEARCHER.searcher.deletedDocs\":5,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":2,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":2,\n    \"__bytes__\":1,\n    \"shard\":\"shard1\",\n    \"collection\":\"testMergeIntegration_collection\"}}",
   [junit4]   2>         "{\"core_node3\":{\n    \"core\":\"testMergeIntegration_collection_shard2_replica_n3\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":3,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":3,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":3,\n    \"__bytes__\":1,\n    \"shard\":\"shard2\",\n    \"collection\":\"testMergeIntegration_collection\"}}"]},
   [junit4]   2>     "_enqueue_time_":24303296322753223,
   [junit4]   2>     "requestedOps":["Op{action=MERGESHARDS, hints={COLL_SHARD=[{\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard2\"}, {\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 1249760 INFO  (AutoscalingActionExecutor-5462-thread-1) [    ] o.a.s.c.a.IndexSizeTriggerTest =======> CapturedEvent{timestamp=24303296516605023, stage=BEFORE_ACTION, actionName='compute_plan', event={
   [junit4]   2>   "id":"5657b6d07d6eebT55f5hrcejmbm8d6y6bt6w6cxf",
   [junit4]   2>   "source":"index_size_trigger",
   [junit4]   2>   "eventTime":24303290690662123,
   [junit4]   2>   "eventType":"INDEXSIZE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "aboveSize":{},
   [junit4]   2>     "belowSize":{"testMergeIntegration_collection":[
   [junit4]   2>         "{\"core_node1\":{\n    \"core\":\"testMergeIntegration_collection_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":7,\n    \"SEARCHER.searcher.deletedDocs\":5,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":2,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":2,\n    \"__bytes__\":1,\n    \"shard\":\"shard1\",\n    \"collection\":\"testMergeIntegration_collection\"}}",
   [junit4]   2>         "{\"core_node3\":{\n    \"core\":\"testMergeIntegration_collection_shard2_replica_n3\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":3,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"__docs__\":3,\n    \"violationType\":\"belowDocs\",\n    \"node_name\":\"127.0.0.1:10005_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"SEARCHER.searcher.numDocs\":3,\n    \"__bytes__\":1,\n    \"shard\":\"shard2\",\n    \"collection\":\"testMergeIntegration_collection\"}}"]},
   [junit4]   2>     "_enqueue_time_":24303296322753223,
   [junit4]   2>     "requestedOps":["Op{action=MERGESHARDS, hints={COLL_SHARD=[{\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard2\"}, {\n  \"first\":\"testMergeIntegration_collection\",\n  \"second\":\"shard1\"}]}}"]}}, context={properties.BEFORE_ACTION=[compute_plan], source=index_size_trigger}, config={
   [junit4]   2>   "trigger":"index_size_trigger",
   [junit4]   2>   "stage":[
   [junit4]   2>     "STARTED",
   [junit4]   2>     "ABORTED",
   [junit4]   2>     "SUCCEEDED",
   [junit4]   2>     "FAILED"],
   [junit4]   2>   "afterAction":[
   [junit4]   2>     "compute_plan",
   [junit4]   2>     "execute_plan"],
   [junit4]   2>   "class":"org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest$CapturingTriggerListener",
   [junit4]   2>   "beforeAction":[
   [junit4]   2>     "compute_plan",
   [junit4]   2>     "execute_plan"]}, message='null'}
   [junit4]   2> 1249760 DEBUG (AutoscalingActionExecutor-5462-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
&nb