[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 54 - Unstable

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
10 messages Options
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 54 - Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/54/

4 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic

Error Message:
there should be new MOVERPLICA ops

Stack Trace:
java.lang.AssertionError: there should be new MOVERPLICA ops
        at __randomizedtesting.SeedInfo.seed([1984A90B71BD0570:B27EB41EAE61835E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic(TestLargeCluster.java:225)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState

Error Message:
Two TriggerAction instances should have been created by now

Stack Trace:
java.lang.AssertionError: Two TriggerAction instances should have been created by now
        at __randomizedtesting.SeedInfo.seed([1984A90B71BD0570:327B7C50EBC510A0]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState(TestTriggerIntegration.java:318)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling

Error Message:
Both triggers should have fired by now

Stack Trace:
java.lang.AssertionError: Both triggers should have fired by now
        at __randomizedtesting.SeedInfo.seed([1984A90B71BD0570:E2A6012EA317E6E2]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling(TestTriggerIntegration.java:226)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([1984A90B71BD0570:F4443E1BCCD198A9]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:1024)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14393 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 3466785 INFO  (SUITE-TestTriggerIntegration-seed#[1984A90B71BD0570]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_1984A90B71BD0570-001/init-core-data-001
   [junit4]   2> 3466787 INFO  (SUITE-TestTriggerIntegration-seed#[1984A90B71BD0570]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3466800 INFO  (SUITE-TestTriggerIntegration-seed#[1984A90B71BD0570]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3466835 INFO  (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 3466835 INFO  (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3466836 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3466840 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3466840 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 3466841 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 3466841 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 3466864 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr]
   [junit4]   2> 3466872 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3466872 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 3466872 DEBUG (ScheduledTrigger-9346-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466896 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466920 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466944 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466948 DEBUG (simCloudManagerPool-9345-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3466948 DEBUG (simCloudManagerPool-9345-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 3466949 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 3466950 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3466950 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 3466968 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466988 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3466989 DEBUG (simCloudManagerPool-9345-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for collection1 / shard1: {"core_node1":{
   [junit4]   2>     "core":"collection1_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10026_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"collection1"}}
   [junit4]   2> 3467008 DEBUG (ScheduledTrigger-9346-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467014 DEBUG (simCloudManagerPool-9345-thread-6) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467014 DEBUG (simCloudManagerPool-9345-thread-6) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 3467014 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3467014 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467014 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3467018 DEBUG (simCloudManagerPool-9345-thread-7) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467018 DEBUG (simCloudManagerPool-9345-thread-7) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 3467018 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 3467018 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467019 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 3467032 DEBUG (ScheduledTrigger-9346-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467056 DEBUG (ScheduledTrigger-9346-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467062 DEBUG (ScheduledTrigger-9346-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5bb1f09a25aaa9Tc1xc33nvqq3wm6qhtnsnfhz5i",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809869817096873,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 3467062 DEBUG (ScheduledTrigger-9346-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, search_rate_trigger]
   [junit4]   2> 3467064 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5bb1f09a25aaa9Tc1xc33nvqq3wm6qhtnsnfhz5i",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809869817096873,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809870947769573,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 3467064 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"5bb1f09a25aaa9Tc1xc33nvqq3wm6qhtnsnfhz5i",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809869817096873,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809870947769573,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}} with context properties: {BEFORE_ACTION=[compute]}
   [junit4]   2> 3467065 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> 3467065 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10025_solr&type=NRT
   [junit4]   2> 3467073 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10025_solr&type=NRT
   [junit4]   2> 3467074 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 25809871500 sessionWrapper.createTime 25809871051451273, this.sessionWrapper.createTime 25809871051451273
   [junit4]   2> 3467074 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"5bb1f09a25aaa9Tc1xc33nvqq3wm6qhtnsnfhz5i",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809869817096873,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "__start__":3,
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809870947769573,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@27051465, org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@29549fd9, org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@76f6dbdc], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
   [junit4]   2> 3467075 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> 3467076 INFO  (simCloudManagerPool-9345-thread-9) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10026_solr for creating new replica
   [junit4]   2> 3467087 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10025_solr&type=NRT
   [junit4]   2> 3467088 INFO  (simCloudManagerPool-9345-thread-13) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10025_solr for creating new replica
   [junit4]   2> 3467111 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10025_solr&type=NRT
   [junit4]   2> 3467113 INFO  (simCloudManagerPool-9345-thread-17) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10025_solr for creating new replica
   [junit4]   2> 3467125 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3467125 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: search_rate_trigger after 100ms
   [junit4]   2> 3467125 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 3051 ms for event id=5bb1f09a25aaa9Tc1xc33nvqq3wm6qhtnsnfhz5i
   [junit4]   2> 3467231 INFO  (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
   [junit4]   2> 3467231 INFO  (TEST-TestTriggerIntegration.testSearchRate-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 2
   [junit4]   2> ## Empty nodes: 0
   [junit4]   2> ## Dead nodes: 0
   [junit4]   2> ## Collections: [collection1]
   [junit4]   2> ## Max replicas per node: 3
   [junit4]   2> ## Min replicas per node: 2
   [junit4]   2> ## Total replicas: 5
   [junit4]   2> ## * collection1 5
   [junit4]   2> ## - active           5
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - ADDREPLICA         3
   [junit4]   2> ## - CREATE             1
   [junit4]   2> ## - DELETESTATUS       3
   [junit4]   2> ## - REQUESTSTATUS      3
   [junit4]   2> ## - autoscaling        3
   [junit4]   2> ## - update             2
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: search_rate_trigger
   [junit4]   2> ## - STARTED         1
   [junit4]   2> ## - SUCCEEDED       1
   [junit4]   2>
   [junit4]   2> 3467232 DEBUG (ScheduledTrigger-9346-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467245 DEBUG (ScheduledTrigger-9346-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 3467245 DEBUG (ScheduledTrigger-9346-thread-4) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, search_rate_trigger]
   [junit4]   2> 3467248 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809880097694423,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> 3467248 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809880097694423,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}} with context properties: {BEFORE_ACTION=[compute]}
   [junit4]   2> 3467249 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> 3467249 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> 3467249 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10025_solr&type=NRT
   [junit4]   2> 3467249 INFO  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 25809880287 sessionWrapper.createTime 25809880257085673, this.sessionWrapper.createTime 25809880257085673
   [junit4]   2> 3467249 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "__start__":3,
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809880097694423,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@605b7b54, org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@3b2602a3, org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@78a5f266], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
   [junit4]   2> 3467250 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=ADDREPLICA&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> 3467262 INFO  (simCloudManagerPool-9345-thread-23) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10026_solr for creating new replica
   [junit4]   2> 3467272 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 3467272 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 3467272 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 3467272 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3467273 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467273 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 3467288 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3467288 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3467288 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3467288 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3467289 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3467292 WARN  (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> java.lang.Exception: Error executing action: execute for trigger event: {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "__start__":3,
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809880097694423,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:312) ~[java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected exception while processing event: {
   [junit4]   2>   "id":"5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":25809870890757073,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "hotReplicas":[
   [junit4]   2>       "{\"core_node2\":{\n    \"core\":\"collection1_shard1_replica_n2\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10025_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    \"core\":\"collection1_shard1_replica_n1\",\n    \"leader\":\"true\",\n    \"SEARCHER.searcher.maxDoc\":0,\n    \"SEARCHER.searcher.deletedDocs\":0,\n    \"INDEX.sizeInBytes\":1,\n    \"rate\":250.0,\n    \"node_name\":\"127.0.0.1:10026_solr\",\n    \"state\":\"active\",\n    \"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250,\n    \"SEARCHER.searcher.numDocs\":0,\n    \"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "__start__":3,
   [junit4]   2>     "coldNodes":{},
   [junit4]   2>     "coldShards":{},
   [junit4]   2>     "coldReplicas":[],
   [junit4]   2>     "coldCollections":{},
   [junit4]   2>     "hotShards":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "hotCollections":{"collection1":500.0},
   [junit4]   2>     "hotNodes":{
   [junit4]   2>       "127.0.0.1:10025_solr":250.0,
   [junit4]   2>       "127.0.0.1:10026_solr":250.0},
   [junit4]   2>     "_enqueue_time_":25809880097694423,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n  \"first\":\"collection1\",\n  \"second\":\"shard1\"}]}}"]}}
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) ~[java/:?]
   [junit4]   2> ... 6 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=ADDREPLICA&async=search_rate_trigger/5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l/0&waitForFinalState=true&collection=collection1&shard=shard1&node=127.0.0.1:10026_solr&type=NRT
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:110) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) ~[java/:?]
   [junit4]   2> ... 6 more
   [junit4]   2> Caused by: java.io.IOException: java.lang.InterruptedException
   [junit4]   2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:540) ~[test/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:80) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) ~[java/:?]
   [junit4]   2> ... 6 more
   [junit4]   2> Caused by: java.lang.InterruptedException
   [junit4]   2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:538) ~[test/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:80) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) ~[java/:?]
   [junit4]   2> ... 6 more
   [junit4]   2> 3467293 DEBUG (AutoscalingActionExecutor-9347-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 2284 ms for event id=5bb1f0da246bd1Tc1xc33nvqq3wm6qhtnsnfhz5l
   [junit4]   2> 3467293 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3467294 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3467294 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3467296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3467296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 3467296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 3467296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 3467296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr]
   [junit4]   2> 3467297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 3467297 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467328 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467348 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467368 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467389 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467409 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467424 DEBUG (simCloudManagerPool-9348-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467425 DEBUG (simCloudManagerPool-9348-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 3467425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 3467426 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467426 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 3467429 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467450 DEBUG (ScheduledTrigger-9349-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 3467465 DEBUG (simCloudManagerPool-9348-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467465 DEBUG (simCloudManagerPool-9348-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 3467465 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 3467466 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3467466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr, 127.0.0.1:10027_solr]
   [junit4]   2> 3467466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 3467466 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 3
   [junit4]   2> 3467476 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 3467504 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 3467504 DEBUG (ScheduledTrigger-9349-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467504 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10027_solr
   [junit4]   2> 3467504 DEBUG (ScheduledTrigger-9349-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10027_solr
   [junit4]   2> 3467528 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 3467528 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467540 DEBUG (simCloudManagerPool-9348-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467540 DEBUG (simCloudManagerPool-9348-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 3467541 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 3467673 DEBUG (ScheduledTrigger-9349-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467673 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 3467673 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 2
   [junit4]   2> ## Empty nodes: 2
   [junit4]   2> ## Dead nodes: 1
   [junit4]   2> ## 127.0.0.1:10027_solr
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        3
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: search_rate_trigger
   [junit4]   2> ## - FAILED          1
   [junit4]   2>
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeLostTriggerRestoreState -Dtests.seed=1984A90B71BD0570 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=es-AR -Dtests.timezone=Africa/Maputo -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4]   2> 3467676 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3467676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr]
   [junit4]   2> 3467676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10027_solr
   [junit4]   2> 3467676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 3467676 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 3467676 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10027_solr]
   [junit4]   2> 3467676 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5bb1f801521d19Tc1xc33nvqq3wm6qhtnsnfhz5o",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":25809901612834073,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809901612834073],
   [junit4]   2>     "nodeNames":["127.0.0.1:10027_solr"]}}
   [junit4] FAILURE 0.44s J2 | TestTriggerIntegration.testNodeLostTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Two TriggerAction instances should have been created by now
   [junit4]    > at __randomizedtesting.SeedInfo.seed([1984A90B71BD0570:327B7C50EBC510A0]:0)
   [junit4]    > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState(TestTriggerIntegration.java:318)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3467692 DEBUG (ScheduledTrigger-9349-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_restore_trigger]
   [junit4]   2> 3467700 DEBUG (AutoscalingActionExecutor-9350-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5bb1f801521d19Tc1xc33nvqq3wm6qhtnsnfhz5o",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":25809901612834073,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809901612834073],
   [junit4]   2>     "_enqueue_time_":25809902426925023,
   [junit4]   2>     "nodeNames":["127.0.0.1:10027_solr"]}}
   [junit4]   2> 3467716 DEBUG (AutoscalingActionExecutor-9350-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3467716 DEBUG (AutoscalingActionExecutor-9350-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_restore_trigger after 100ms
   [junit4]   2> 3467716 DEBUG (AutoscalingActionExecutor-9350-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 796 ms for event id=5bb1f801521d19Tc1xc33nvqq3wm6qhtnsnfhz5o
   [junit4]   2> 3467757 INFO  (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCooldown
   [junit4]   2> 3467757 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 3467757 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10027_solr
   [junit4]   2> 3467757 INFO  (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3467757 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3467760 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3467760 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3467760 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3467768 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3467768 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3467768 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3467771 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3467771 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 3467772 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion -1
   [junit4]   2> 3467772 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 3467784 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr]
   [junit4]   2> 3467784 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467784 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 3467793 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467824 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467848 DEBUG (ScheduledTrigger-9352-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467872 DEBUG (ScheduledTrigger-9352-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467886 DEBUG (simCloudManagerPool-9351-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467886 DEBUG (simCloudManagerPool-9351-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
   [junit4]   2> 3467887 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 3467887 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3467887 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 3467890 DEBUG (simCloudManagerPool-9351-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467891 DEBUG (simCloudManagerPool-9351-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
   [junit4]   2> 3467892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 3467892 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3467892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger - Initial livenodes: [127.0.0.1:10025_solr, 127.0.0.1:10026_solr]
   [junit4]   2> 3467892 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 3467893 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 3467893 DEBUG (ScheduledTrigger-9352-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3467893 DEBUG (ScheduledTrigger-9352-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 3467913 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3467914 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3467914 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 3467914 DEBUG (simCloudManagerPool-9351-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3467914 DEBUG (simCloudManagerPool-9351-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
   [junit4]   2> 3467915 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 3467915 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 3467933 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 3467934 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3467934 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 3467934 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10028_solr at time 25809914518274723
   [junit4]   2> 3467953 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 3467954 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3467954 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 3467954 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10028_solr] added at times [25809914518274723], now=25809915530189223
   [junit4]   2> 3467954 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5bb1fb028b82a3Tc1xc33nvqq3wm6qhtnsnfhz5r",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":25809914518274723,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809914518274723],
   [junit4]   2>     "nodeNames":["127.0.0.1:10028_solr"]}}
   [junit4]   2> 3467955 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_cooldown_trigger]
   [junit4]   2> 3467968 DEBUG (ScheduledTrigger-9352-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10028_solr
   [junit4]   2> 3467968 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5bb1fb028b82a3Tc1xc33nvqq3wm6qhtnsnfhz5r",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":25809914518274723,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809914518274723],
   [junit4]   2>     "_enqueue_time_":25809915561044323,
   [junit4]   2>     "nodeNames":["127.0.0.1:10028_solr"]}}
   [junit4]   2> 3467976 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3467976 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_cooldown_trigger after 100ms
   [junit4]   2> 3467976 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 386 ms for event id=5bb1fb028b82a3Tc1xc33nvqq3wm6qhtnsnfhz5r
   [junit4]   2> 3468130 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 3468131 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3468131 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 3468131 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10029_solr at time 25809924352099273
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10029_solr] added at times [25809924352099273], now=25809925362224023
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"5bb1fd4cafc3c9Tc1xc33nvqq3wm6qhtnsnfhz5u",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":25809924352099273,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809924352099273],
   [junit4]   2>     "nodeNames":["127.0.0.1:10029_solr"]}}
   [junit4]   2> 3468151 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_cooldown_trigger]
   [junit4]   2> 3468176 DEBUG (ScheduledTrigger-9352-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10029_solr
   [junit4]   2> 3468176 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"5bb1fd4cafc3c9Tc1xc33nvqq3wm6qhtnsnfhz5u",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":25809924352099273,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[25809924352099273],
   [junit4]   2>     "_enqueue_time_":25809925375350973,
   [junit4]   2>     "nodeNames":["127.0.0.1:10029_solr"]}}
   [junit4]   2> 3468180 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3468180 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_cooldown_trigger after 100ms
   [junit4]   2> 3468180 DEBUG (AutoscalingActionExecutor-9353-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 210 ms for event id=5bb1fd4cafc3c9Tc1xc33nvqq3wm6qhtnsnfhz5u
   [junit4]   2> 3468217 INFO  (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCooldown
   [junit4]   2> 3468217 INFO  (TEST-TestTriggerIntegration.testCooldown-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
   [junit4]   2> ############ CLUSTER STATE ############
   [junit4]   2> #######################################
   [junit4]   2> ## Live nodes: 4
   [junit4]   2> ## Empty nodes: 4
   [junit4]   2> ## Dead nodes: 0
   [junit4]   2> ## Collections: []
   [junit4]   2> ## Max replicas per node: 0
   [junit4]   2> ## Min replicas per node: 0
   [junit4]   2> ## Total replicas: 0
   [junit4]   2> ######### Solr op counts ##########
   [junit4]   2> ## - autoscaling        3
   [junit4]   2> ## - update             4
   [junit4]   2> ######### Autoscaling event counts ###########
   [junit4]   2> ## * Trigger: node_added_cooldown_trigger
   [junit4]   2> ## - STARTED         2
   [junit4]   2> ## - SUCCEEDED       2
   [junit4]   2>
   [junit4]   2> 3468248 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 3468248 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
   [junit4]   2> 3468249 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3468249 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3468252 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3468256 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3468256 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3468256 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3468256 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3468256 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[1984A90B71BD0570]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3468278 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3468278 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
   [junit4]   2> 3468280 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion -1
   [junit4]   2> 3468280 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 3468291 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10029_solr, 127.0.0.1:10028_solr]
   [junit4]   2> 3468308 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3468308 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 3468308 DEBUG (ScheduledTrigger-9355-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3468328 DEBUG (ScheduledTrigger-9355-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 3468348 DEBUG (ScheduledTrigger-9355-thread-1) [    ] o.a.s.c.a.Nod

[...truncated too long message...]

owed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1785286877
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1785286877
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1785286877

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null165047496
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null165047496
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null165047496

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 31ms :: artifacts dl 2ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 159 minutes 34 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 55 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/55/

3 tests failed.
FAILED:  org.apache.solr.cloud.TestTlogReplica.testRecovery

Error Message:
Can not find doc 8 in https://127.0.0.1:36274/solr

Stack Trace:
java.lang.AssertionError: Can not find doc 8 in https://127.0.0.1:36274/solr
        at __randomizedtesting.SeedInfo.seed([CC3F7B9D23DD3C7:CD338E15FF6D1960]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.solr.cloud.TestTlogReplica.checkRTG(TestTlogReplica.java:889)
        at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:603)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testTrigger

Error Message:
should have fired an event

Stack Trace:
java.lang.AssertionError: should have fired an event
        at __randomizedtesting.SeedInfo.seed([CC3F7B9D23DD3C7:6F08C13B4BF2A0EA]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testTrigger(IndexSizeTriggerTest.java:184)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testSplitIntegration

Error Message:
last state: DocCollection(testSplitIntegration_collection//clusterstate.json/91)={   "replicationFactor":"2",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"2",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0",   "autoCreated":"true",   "shards":{     "shard2":{       "replicas":{         "core_node3":{           "core":"testSplitIntegration_collection_shard2_replica_n3",           "leader":"true",           "SEARCHER.searcher.maxDoc":11,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":11},         "core_node4":{           "core":"testSplitIntegration_collection_shard2_replica_n4",           "SEARCHER.searcher.maxDoc":11,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":11}},       "range":"0-7fffffff",       "state":"active"},     "shard1":{       "stateTimestamp":"1525760846274004350",       "replicas":{         "core_node1":{           "core":"testSplitIntegration_collection_shard1_replica_n1",           "leader":"true",           "SEARCHER.searcher.maxDoc":14,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":14},         "core_node2":{           "core":"testSplitIntegration_collection_shard1_replica_n2",           "SEARCHER.searcher.maxDoc":14,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":14}},       "range":"80000000-ffffffff",       "state":"inactive"},     "shard1_1":{       "parent":"shard1",       "stateTimestamp":"1525760846306640600",       "range":"c0000000-ffffffff",       "state":"active",       "replicas":{         "core_node10":{           "leader":"true",           "core":"testSplitIntegration_collection_shard1_1_replica1",           "SEARCHER.searcher.maxDoc":6,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "base_url":"http://127.0.0.1:10000/solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":6},         "core_node9":{           "core":"testSplitIntegration_collection_shard1_1_replica0",           "SEARCHER.searcher.maxDoc":6,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "base_url":"http://127.0.0.1:10001/solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":6}}},     "shard1_0":{       "parent":"shard1",       "stateTimestamp":"1525760846306232750",       "range":"80000000-bfffffff",       "state":"active",       "replicas":{         "core_node7":{           "leader":"true",           "core":"testSplitIntegration_collection_shard1_0_replica0",           "SEARCHER.searcher.maxDoc":7,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "base_url":"http://127.0.0.1:10001/solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":7},         "core_node8":{           "core":"testSplitIntegration_collection_shard1_0_replica1",           "SEARCHER.searcher.maxDoc":7,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "base_url":"http://127.0.0.1:10000/solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":7}}}}}

Stack Trace:
java.util.concurrent.TimeoutException: last state: DocCollection(testSplitIntegration_collection//clusterstate.json/91)={
  "replicationFactor":"2",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "shards":{
    "shard2":{
      "replicas":{
        "core_node3":{
          "core":"testSplitIntegration_collection_shard2_replica_n3",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":11,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10000_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":11},
        "core_node4":{
          "core":"testSplitIntegration_collection_shard2_replica_n4",
          "SEARCHER.searcher.maxDoc":11,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10001_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":11}},
      "range":"0-7fffffff",
      "state":"active"},
    "shard1":{
      "stateTimestamp":"1525760846274004350",
      "replicas":{
        "core_node1":{
          "core":"testSplitIntegration_collection_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":14,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10000_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":14},
        "core_node2":{
          "core":"testSplitIntegration_collection_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":14,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10001_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":14}},
      "range":"80000000-ffffffff",
      "state":"inactive"},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1525760846306640600",
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testSplitIntegration_collection_shard1_1_replica1",
          "SEARCHER.searcher.maxDoc":6,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10000_solr",
          "base_url":"http://127.0.0.1:10000/solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":6},
        "core_node9":{
          "core":"testSplitIntegration_collection_shard1_1_replica0",
          "SEARCHER.searcher.maxDoc":6,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10001_solr",
          "base_url":"http://127.0.0.1:10001/solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":6}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1525760846306232750",
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "leader":"true",
          "core":"testSplitIntegration_collection_shard1_0_replica0",
          "SEARCHER.searcher.maxDoc":7,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10001_solr",
          "base_url":"http://127.0.0.1:10001/solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":7},
        "core_node8":{
          "core":"testSplitIntegration_collection_shard1_0_replica1",
          "SEARCHER.searcher.maxDoc":7,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10000_solr",
          "base_url":"http://127.0.0.1:10000/solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":7}}}}}
        at __randomizedtesting.SeedInfo.seed([CC3F7B9D23DD3C7:354D4EF9FDC21A39]:0)
        at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:111)
        at org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testSplitIntegration(IndexSizeTriggerTest.java:299)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12534 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTlogReplica
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/init-core-data-001
   [junit4]   2> 353834 WARN  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 353834 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 353842 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 353843 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 353844 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001
   [junit4]   2> 353844 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 353844 INFO  (Thread-2328) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 353844 INFO  (Thread-2328) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 353847 ERROR (Thread-2328) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 353944 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:40678
   [junit4]   2> 353948 INFO  (zkConnectionManagerCallback-367-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 353955 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 353958 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 353960 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 353960 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 353960 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 353961 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 353961 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 353961 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 353961 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1613ab2b{/solr,null,AVAILABLE}
   [junit4]   2> 353962 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f9e4ded{/solr,null,AVAILABLE}
   [junit4]   2> 353963 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f5bb8da{SSL,[ssl, http/1.1]}{127.0.0.1:36274}
   [junit4]   2> 353963 INFO  (jetty-launcher-364-thread-2) [    ] o.e.j.s.Server Started @354075ms
   [junit4]   2> 353963 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36274}
   [junit4]   2> 353964 ERROR (jetty-launcher-364-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 353964 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 353964 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.4.0
   [junit4]   2> 353964 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 353964 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 353964 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-08T06:13:27.685Z
   [junit4]   2> 353978 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@111d052a{SSL,[ssl, http/1.1]}{127.0.0.1:60535}
   [junit4]   2> 353978 INFO  (jetty-launcher-364-thread-1) [    ] o.e.j.s.Server Started @354090ms
   [junit4]   2> 353978 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60535}
   [junit4]   2> 353979 ERROR (jetty-launcher-364-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 353979 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 353979 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.4.0
   [junit4]   2> 353979 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 353979 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 353979 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-08T06:13:27.700Z
   [junit4]   2> 353989 INFO  (zkConnectionManagerCallback-369-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 353990 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 354003 INFO  (zkConnectionManagerCallback-371-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354003 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 354014 INFO  (jetty-launcher-364-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40678/solr
   [junit4]   2> 354017 INFO  (zkConnectionManagerCallback-375-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354071 INFO  (zkConnectionManagerCallback-377-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354085 INFO  (jetty-launcher-364-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40678/solr
   [junit4]   2> 354102 INFO  (zkConnectionManagerCallback-383-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354106 INFO  (zkConnectionManagerCallback-385-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354126 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 354127 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60535_solr
   [junit4]   2> 354129 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.Overseer Overseer (id=73762310218252294-127.0.0.1:60535_solr-n_0000000000) starting
   [junit4]   2> 354142 INFO  (zkConnectionManagerCallback-392-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354145 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40678/solr ready
   [junit4]   2> 354169 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60535_solr
   [junit4]   2> 354172 INFO  (OverseerStateUpdate-73762310218252294-127.0.0.1:60535_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 354256 INFO  (zkCallback-391-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 354389 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 354399 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 354402 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 354402 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36274_solr
   [junit4]   2> 354404 INFO  (zkCallback-391-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 354406 INFO  (zkCallback-384-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 354446 INFO  (zkCallback-376-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 354691 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60535.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354729 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60535.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354729 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60535.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354735 INFO  (jetty-launcher-364-thread-1) [n:127.0.0.1:60535_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node1/.
   [junit4]   2> 354827 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36274.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354838 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36274.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354838 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36274.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 354842 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node2/.
   [junit4]   2> 354858 INFO  (zkConnectionManagerCallback-397-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 354859 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 354861 INFO  (jetty-launcher-364-thread-2) [n:127.0.0.1:36274_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40678/solr ready
   [junit4]   2> 355150 INFO  (zkConnectionManagerCallback-399-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 355178 INFO  (zkConnectionManagerCallback-404-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 355179 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 355181 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40678/solr ready
   [junit4]   2> 355184 INFO  (SUITE-TestTlogReplica-seed#[CC3F7B9D23DD3C7]-worker) [    ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
   [junit4]   2> 355272 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 355278 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 355322 INFO  (TEST-TestTlogReplica.testRemoveLeader-seed#[CC3F7B9D23DD3C7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRemoveLeader
   [junit4]   2> 355326 INFO  (qtp1108498330-3578) [n:127.0.0.1:36274_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_remove_leader&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 355332 INFO  (OverseerThreadFactory-561-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection tlog_replica_test_remove_leader
   [junit4]   2> 355454 INFO  (OverseerStateUpdate-73762310218252294-127.0.0.1:60535_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_remove_leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_remove_leader_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:60535/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 355459 INFO  (OverseerStateUpdate-73762310218252294-127.0.0.1:60535_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_remove_leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_remove_leader_shard1_replica_t3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36274/solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 355694 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t3&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 355703 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr    x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 355704 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr    x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 356720 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 356731 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.IndexSchema [tlog_replica_test_remove_leader_shard1_replica_t3] Schema name=minimal
   [junit4]   2> 356731 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 356738 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 356738 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_remove_leader_shard1_replica_t3' using configuration from collection tlog_replica_test_remove_leader, trusted=true
   [junit4]   2> 356738 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36274.solr.core.tlog_replica_test_remove_leader.shard1.replica_t3' (registry 'solr.core.tlog_replica_test_remove_leader.shard1.replica_t3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 356739 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 356739 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrCore [[tlog_replica_test_remove_leader_shard1_replica_t3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node2/tlog_replica_test_remove_leader_shard1_replica_t3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node2/./tlog_replica_test_remove_leader_shard1_replica_t3/data/]
   [junit4]   2> 356742 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_remove_leader_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 356746 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 356747 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_remove_leader_shard1_replica_t1' using configuration from collection tlog_replica_test_remove_leader, trusted=true
   [junit4]   2> 356747 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60535.solr.core.tlog_replica_test_remove_leader.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_remove_leader.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e0b8018
   [junit4]   2> 356747 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 356747 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_remove_leader_shard1_replica_t1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node1/tlog_replica_test_remove_leader_shard1_replica_t1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_CC3F7B9D23DD3C7-001/tempDir-001/node1/./tlog_replica_test_remove_leader_shard1_replica_t1/data/]
   [junit4]   2> 356845 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 356845 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 356846 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 356846 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 356849 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.SolrIndexSearcher Opening [Searcher@267752ac[tlog_replica_test_remove_leader_shard1_replica_t3] main]
   [junit4]   2> 356851 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 356852 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 356853 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 356853 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 356853 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 356855 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 356855 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 356857 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1599875328851836928
   [junit4]   2> 356860 INFO  (searcherExecutor-566-thread-1-processing-n:127.0.0.1:36274_solr x:tlog_replica_test_remove_leader_shard1_replica_t3 c:tlog_replica_test_remove_leader s:shard1 r:core_node4) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t3] Registered new searcher Searcher@267752ac[tlog_replica_test_remove_leader_shard1_replica_t3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 356866 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tlog_replica_test_remove_leader/terms/shard1 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 356866 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d84a742[tlog_replica_test_remove_leader_shard1_replica_t1] main]
   [junit4]   2> 356868 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 356868 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 356869 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 356869 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1599875328864419840
   [junit4]   2> 356873 INFO  (searcherExecutor-567-thread-1-processing-n:127.0.0.1:60535_solr x:tlog_replica_test_remove_leader_shard1_replica_t1 c:tlog_replica_test_remove_leader s:shard1 r:core_node2) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t1] Registered new searcher Searcher@4d84a742[tlog_replica_test_remove_leader_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 356878 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tlog_replica_test_remove_leader/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=1}
   [junit4]   2> 356879 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9998ms
   [junit4]   2> 357383 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 357383 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 357383 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/
   [junit4]   2> 357384 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_remove_leader_shard1_replica_t3 url=https://127.0.0.1:36274/solr START replicas=[https://127.0.0.1:60535/solr/tlog_replica_test_remove_leader_shard1_replica_t1/] nUpdates=100
   [junit4]   2> 357401 INFO  (qtp619106724-3566) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 357685 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_remove_leader_shard1_replica_t3 url=https://127.0.0.1:36274/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 357685 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 357685 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 357685 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 357686 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkController tlog_replica_test_remove_leader_shard1_replica_t3 stopping background replication from leader
   [junit4]   2> 357690 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/ shard1
   [junit4]   2> 357794 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 357797 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t3&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2103
   [junit4]   2> 357885 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_remove_leader_shard1_replica_t1 starting background replication from leader
   [junit4]   2> 357885 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 357888 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 357888 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 357891 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=tlog_replica_test_remove_leader_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2188
   [junit4]   2> 357896 INFO  (qtp1108498330-3578) [n:127.0.0.1:36274_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 357992 INFO  (zkCallback-376-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 357992 INFO  (zkCallback-384-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 358896 INFO  (qtp1108498330-3578) [n:127.0.0.1:36274_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_remove_leader&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=3570
   [junit4]   2> 358993 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tlog_replica_test_remove_leader/terms/shard1 to Terms{values={core_node2=1, core_node4=1}, version=2}
   [junit4]   2> 358996 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/&wt=javabin&version=2}{add=[1 (1599875331041263616)]} 0 1
   [junit4]   2> 358996 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1599875331041263616)]} 0 58
   [junit4]   2> 358999 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 359020 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1599875331119906816,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 359020 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@109076b3 commitCommandVersion:1599875331119906816
   [junit4]   2> 359035 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.SolrIndexSearcher Opening [Searcher@59e2f122[tlog_replica_test_remove_leader_shard1_replica_t3] main]
   [junit4]   2> 359037 INFO  (searcherExecutor-566-thread-1-processing-n:127.0.0.1:36274_solr x:tlog_replica_test_remove_leader_shard1_replica_t3 c:tlog_replica_test_remove_leader s:shard1 r:core_node4) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t3] Registered new searcher Searcher@59e2f122[tlog_replica_test_remove_leader_shard1_replica_t3] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1)))}
   [junit4]   2> 359037 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 359038 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 17
   [junit4]   2> 359038 INFO  (qtp1108498330-3578) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/update params={_stateVer_=tlog_replica_test_remove_leader:5&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 40
   [junit4]   2> 359051 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 359068 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359171 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359273 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359334 INFO  (OverseerCollectionConfigSetProcessor-73762310218252294-127.0.0.1:60535_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 359375 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359477 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359578 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359680 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359783 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359885 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 359988 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360099 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360201 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360304 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360406 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360508 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360541 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3/
   [junit4]   2> 360549 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 360549 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 360549 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 1525760012741
   [junit4]   2> 360549 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 360549 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 360549 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 360557 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=7
   [junit4]   2> 360558 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Number of files in latest index in master: 19
   [junit4]   2> 360560 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@77d7b283 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3a336b91)
   [junit4]   2> 360566 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360576 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360582 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360584 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360587 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360588 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360590 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360592 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360593 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360595 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360596 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360598 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360599 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360609 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.sd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360646 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 360647 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360649 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.dvdm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360651 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.tfp&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360656 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360657 INFO  (qtp1108498330-3568) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 360658 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Bytes downloaded: 2994, Bytes skipped downloading: 0
   [junit4]   2> 360658 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=2994) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@77d7b283 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3a336b91)
   [junit4]   2> 360675 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 360677 INFO  (indexFetcher-577-thread-1) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@33ce144e[tlog_replica_test_remove_leader_shard1_replica_t1] main]
   [junit4]   2> 360679 INFO  (searcherExecutor-567-thread-1) [    ] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t1] Registered new searcher Searcher@33ce144e[tlog_replica_test_remove_leader_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1)))}
   [junit4]   2> 360749 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 360774 INFO  (qtp1108498330-3578) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node4 x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t3]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 360776 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletereplica with params replica=core_node4&action=DELETEREPLICA&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 360786 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tlog_replica_test_remove_leader.shard1.replica_t3, tag=null
   [junit4]   2> 360786 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6d2275b7: rootName = solr_36274, domain = solr.core.tlog_replica_test_remove_leader.shard1.replica_t3, service url = null, agent id = null] for registry solr.core.tlog_replica_test_remove_leader.shard1.replica_t3 / com.codahale.metrics.MetricRegistry@1949f743
   [junit4]   2> 360815 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkController tlog_replica_test_remove_leader_shard1_replica_t3 stopping background replication from leader
   [junit4]   2> 360815 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t3]  CLOSING SolrCore org.apache.solr.core.SolrCore@2f9e1541
   [junit4]   2> 360815 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tlog_replica_test_remove_leader.shard1.replica_t3, tag=2f9e1541
   [junit4]   2> 360815 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.tlog_replica_test_remove_leader.shard1.leader, tag=2f9e1541
   [junit4]   2> 360819 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tlog_replica_test_remove_leader/terms/shard1 to Terms{values={core_node2=1}, version=3}
   [junit4]   2> 360823 INFO  (qtp1108498330-3574) [n:127.0.0.1:36274_solr    x:tlog_replica_test_remove_leader_shard1_replica_t3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=tlog_replica_test_remove_leader_shard1_replica_t3&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2&deleteIndex=true} status=0 QTime=37
   [junit4]   2> 360828 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 360830 INFO  (zkCallback-376-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 360832 INFO  (zkCallback-384-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 360934 INFO  (zkCallback-384-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 361036 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr c:tlog_replica_test_remove_leader   ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replica=core_node4&action=DELETEREPLICA&collection=tlog_replica_test_remove_leader&shard=shard1&wt=javabin&version=2} status=0 QTime=260
   [junit4]   2> 362784 INFO  (OverseerCollectionConfigSetProcessor-73762310218252294-127.0.0.1:60535_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 363328 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60535/solr/tlog_replica_test_remove_leader_shard1_replica_t1/
   [junit4]   2> 363328 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 363328 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SyncStrategy https://127.0.0.1:60535/solr/tlog_replica_test_remove_leader_shard1_replica_t1/ has no replicas
   [junit4]   2> 363329 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_remove_leader_shard1_replica_t1 stopping background replication from leader
   [junit4]   2> 363330 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext New leader does not have old tlog to replay
   [junit4]   2> 363333 INFO  (zkCallback-384-thread-1) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60535/solr/tlog_replica_test_remove_leader_shard1_replica_t1/ shard1
   [junit4]   2> 363335 INFO  (zkCallback-384-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_remove_leader/state.json] for collection [tlog_replica_test_remove_leader] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 363556 ERROR (TEST-TestTlogReplica.testRemoveLeader-seed#[CC3F7B9D23DD3C7]) [    ] o.a.s.c.s.i.CloudSolrClient Request to collection [tlog_replica_test_remove_leader] failed due to (404) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36274/solr/tlog_replica_test_remove_leader_shard1_replica_t3: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]   2> <head>
   [junit4]   2> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
   [junit4]   2> <title>Error 404 Can not find: /solr/tlog_replica_test_remove_leader_shard1_replica_t3/update</title>
   [junit4]   2> </head>
   [junit4]   2> <body><h2>HTTP ERROR 404</h2>
   [junit4]   2> <p>Problem accessing /solr/tlog_replica_test_remove_leader_shard1_replica_t3/update. Reason:
   [junit4]   2> <pre>    Can not find: /solr/tlog_replica_test_remove_leader_shard1_replica_t3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>
   [junit4]   2>
   [junit4]   2> </body>
   [junit4]   2> </html>
   [junit4]   2> , retry? 0
   [junit4]   2> 363568 INFO  (qtp619106724-3571) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1599875335882539008)]} 0 6
   [junit4]   2> 363570 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1599875335890927616,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 363570 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@f7dbee7 commitCommandVersion:1599875335890927616
   [junit4]   2> 363594 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@92f82d0[tlog_replica_test_remove_leader_shard1_replica_t1] main]
   [junit4]   2> 363595 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 363596 INFO  (searcherExecutor-567-thread-1-processing-n:127.0.0.1:60535_solr x:tlog_replica_test_remove_leader_shard1_replica_t1 c:tlog_replica_test_remove_leader s:shard1 r:core_node2) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_remove_leader_shard1_replica_t1] Registered new searcher Searcher@92f82d0[tlog_replica_test_remove_leader_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1)))}
   [junit4]   2> 363597 INFO  (qtp619106724-3577) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/update params={_stateVer_=tlog_replica_test_remove_leader:8&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 26
   [junit4]   2> 363636 INFO  (qtp619106724-3575) [n:127.0.0.1:60535_solr c:tlog_replica_test_remove_leader s:shard1 r:core_node2 x:tlog_replica_test_remove_leader_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_remove_leader_shard1_replica_t1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=2 status=0 QTime=0
   [junit4]   2> 363638 INFO  (qtp1108498330-3580) [n:127.0.0.1:36274_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params action=ADDREPLICA&collection=tlog_replica_test_remove_leader&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 363658 INFO  (OverseerThreadFactory-561-thread-3) [ c:tlog_replica_test_remove_leader s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127

[...truncated too long message...]

wed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null112989872
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null112989872
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null112989872

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1406270619
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1406270619
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1406270619

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 91ms :: artifacts dl 11ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 167 minutes 40 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 56 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/56/

8 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
        at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
        at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([B501811370D20380]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor58.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry:     1) Thread[id=6324, name=OverseerAutoScalingTriggerThread-72597879476912139-dummy.host.com:8984_solr-n_0000000002, state=RUNNABLE, group=Overseer autoscaling triggers]         at sun.reflect.Reflection.getCallerClass(Native Method)         at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source)         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)         at java.lang.reflect.Method.invoke(Method.java:498)         at org.apache.logging.log4j.util.StackLocator.getCallerClass(StackLocator.java:110)         at org.apache.logging.log4j.util.StackLocator.getCurrentStackTrace(StackLocator.java:169)         at org.apache.logging.log4j.util.StackLocatorUtil.getCurrentStackTrace(StackLocatorUtil.java:67)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:137)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)         at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)         at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)         at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)         at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)         at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)         at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)         at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)         at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)         at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)         at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)         at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)         at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)         at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)         at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry:
   1) Thread[id=6324, name=OverseerAutoScalingTriggerThread-72597879476912139-dummy.host.com:8984_solr-n_0000000002, state=RUNNABLE, group=Overseer autoscaling triggers]
        at sun.reflect.Reflection.getCallerClass(Native Method)
        at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.logging.log4j.util.StackLocator.getCallerClass(StackLocator.java:110)
        at org.apache.logging.log4j.util.StackLocator.getCurrentStackTrace(StackLocator.java:169)
        at org.apache.logging.log4j.util.StackLocatorUtil.getCurrentStackTrace(StackLocatorUtil.java:67)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:137)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)
        at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)
        at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)
        at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)
        at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)
        at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
        at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
        at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
        at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)
        at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
        at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)
        at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)
        at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([B501811370D20380]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
        at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
        at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([B501811370D20380]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:     1) Thread[id=21795, name=OverseerAutoScalingTriggerThread-72598060914180098-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]         at java.security.AccessController.getStackAccessControlContext(Native Method)         at java.security.AccessController.checkPermission(AccessController.java:858)         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)         at java.lang.Class.getProtectionDomain(Class.java:2299)         at org.apache.logging.log4j.core.impl.ThrowableProxy.toCacheEntry(ThrowableProxy.java:654)         at org.apache.logging.log4j.core.impl.ThrowableProxy.toExtendedStackTrace(ThrowableProxy.java:738)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)         at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)         at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)         at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)         at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)         at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)         at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)         at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)         at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)         at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)         at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)         at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)         at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)         at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)         at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)         at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)         at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:
   1) Thread[id=21795, name=OverseerAutoScalingTriggerThread-72598060914180098-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
        at java.security.AccessController.getStackAccessControlContext(Native Method)
        at java.security.AccessController.checkPermission(AccessController.java:858)
        at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
        at java.lang.Class.getProtectionDomain(Class.java:2299)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.toCacheEntry(ThrowableProxy.java:654)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.toExtendedStackTrace(ThrowableProxy.java:738)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)
        at org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)
        at org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:605)
        at org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)
        at org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)
        at org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:334)
        at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:233)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:218)
        at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:58)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
        at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
        at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
        at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)
        at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
        at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)
        at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)
        at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([B501811370D20380]:0)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([B501811370D20380:E8499F9ABF14A5CF]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate(TestLargeCluster.java:579)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic

Error Message:
there should be new MOVERPLICA ops

Stack Trace:
java.lang.AssertionError: there should be new MOVERPLICA ops
        at __randomizedtesting.SeedInfo.seed([B501811370D20380:1EFB9C06AF0E85AE]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic(TestLargeCluster.java:225)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners

Error Message:
foo events: [CapturedEvent{timestamp=8488781869099865, stage=STARTED, actionName='null', event={   "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",   "source":"node_added_trigger",   "eventTime":8488778862121415,   "eventType":"NODEADDED",   "properties":{     "eventTimes":[8488778862121415],     "_enqueue_time_":8488781848291115,     "nodeNames":["127.0.0.1:10037_solr"]}}, context={}, config={   "trigger":"node_added_trigger",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "afterAction":[     "test",     "test1"],   "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",   "beforeAction":"test"}, message='null'}, CapturedEvent{timestamp=8488781885751515, stage=BEFORE_ACTION, actionName='test', event={   "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",   "source":"node_added_trigger",   "eventTime":8488778862121415,   "eventType":"NODEADDED",   "properties":{     "eventTimes":[8488778862121415],     "_enqueue_time_":8488781848291115,     "nodeNames":["127.0.0.1:10037_solr"]}}, context={properties.BEFORE_ACTION=[test], source=node_added_trigger}, config={   "trigger":"node_added_trigger",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "afterAction":[     "test",     "test1"],   "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",   "beforeAction":"test"}, message='null'}, CapturedEvent{timestamp=8488781888141065, stage=AFTER_ACTION, actionName='test', event={   "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",   "source":"node_added_trigger",   "eventTime":8488778862121415,   "eventType":"NODEADDED",   "properties":{     "eventTimes":[8488778862121415],     "_enqueue_time_":8488781848291115,     "nodeNames":["127.0.0.1:10037_solr"]}}, context={properties.BEFORE_ACTION=[test], source=node_added_trigger, properties.AFTER_ACTION=[test]}, config={   "trigger":"node_added_trigger",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "afterAction":[     "test",     "test1"],   "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",   "beforeAction":"test"}, message='null'}] expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: foo events: [CapturedEvent{timestamp=8488781869099865, stage=STARTED, actionName='null', event={
  "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",
  "source":"node_added_trigger",
  "eventTime":8488778862121415,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[8488778862121415],
    "_enqueue_time_":8488781848291115,
    "nodeNames":["127.0.0.1:10037_solr"]}}, context={}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}, CapturedEvent{timestamp=8488781885751515, stage=BEFORE_ACTION, actionName='test', event={
  "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",
  "source":"node_added_trigger",
  "eventTime":8488778862121415,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[8488778862121415],
    "_enqueue_time_":8488781848291115,
    "nodeNames":["127.0.0.1:10037_solr"]}}, context={properties.BEFORE_ACTION=[test], source=node_added_trigger}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}, CapturedEvent{timestamp=8488781888141065, stage=AFTER_ACTION, actionName='test', event={
  "id":"1e287fd8a751c7T905ufpvfbxgks2mvapm8ks8a0",
  "source":"node_added_trigger",
  "eventTime":8488778862121415,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[8488778862121415],
    "_enqueue_time_":8488781848291115,
    "nodeNames":["127.0.0.1:10037_solr"]}}, context={properties.BEFORE_ACTION=[test], source=node_added_trigger, properties.AFTER_ACTION=[test]}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}] expected:<4> but was:<3>
        at __randomizedtesting.SeedInfo.seed([B501811370D20380:58C11603CDBE9E59]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:1031)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelComplementStream

Error Message:
java.util.concurrent.ExecutionException: java.io.IOException: --> https://127.0.0.1:45561/solr/collection1_collection_shard2_replica_n2/:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection

Stack Trace:
java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: --> https://127.0.0.1:45561/solr/collection1_collection_shard2_replica_n2/:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
        at __randomizedtesting.SeedInfo.seed([D93A6B967DF6D252:EC44E6BBD3C43BC8]:0)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream.openStreams(CloudSolrStream.java:400)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream.open(CloudSolrStream.java:275)
        at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.getTuples(StreamDecoratorTest.java:3840)
        at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelComplementStream(StreamDecoratorTest.java:3828)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: java.io.IOException: --> https://127.0.0.1:45561/solr/collection1_collection_shard2_replica_n2/:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
        at java.util.concurrent.FutureTask.report(FutureTask.java:122)
        at java.util.concurrent.FutureTask.get(FutureTask.java:192)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream.openStreams(CloudSolrStream.java:394)
        ... 42 more
Caused by: java.io.IOException: --> https://127.0.0.1:45561/solr/collection1_collection_shard2_replica_n2/:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
        at org.apache.solr.client.solrj.io.stream.SolrStream.read(SolrStream.java:222)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream$TupleWrapper.next(CloudSolrStream.java:484)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream$StreamOpener.call(CloudSolrStream.java:507)
        at org.apache.solr.client.solrj.io.stream.CloudSolrStream$StreamOpener.call(CloudSolrStream.java:494)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        ... 1 more




Build Log:
[...truncated 12990 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestLeaderElectionZkExpiry
   [junit4]   2> 1448538 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[B501811370D20380]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestLeaderElectionZkExpiry_B501811370D20380-001/init-core-data-001
   [junit4]   2> 1448555 WARN  (SUITE-TestLeaderElectionZkExpiry-seed#[B501811370D20380]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 1448555 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[B501811370D20380]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1448557 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[B501811370D20380]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1448568 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testLeaderElectionWithZkExpiry
   [junit4]   2> 1448585 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6356814b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1448805 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6356814b
   [junit4]   2> 1448841 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6356814b
   [junit4]   2> 1448841 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6356814b
   [junit4]   2> 1448858 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestLeaderElectionZkExpiry_B501811370D20380-001/testLeaderElectionWithZkExpiry-solr-001
   [junit4]   2> 1448911 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1448911 INFO  (Thread-2604) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1448911 INFO  (Thread-2604) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1448913 ERROR (Thread-2604) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1449011 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [    ] o.a.s.c.ZkTestServer start zk server on port:41331
   [junit4]   2> 1449026 INFO  (zkConnectionManagerCallback-1184-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1449062 INFO  (zkConnectionManagerCallback-1186-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1449078 INFO  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1449417 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1449430 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr
   [junit4]   2> 1449430 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=72597879476912130-dummy.host.com:8984_solr-n_0000000000) starting
   [junit4]   2> 1449471 INFO  (zkConnectionManagerCallback-1195-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1449473 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [n:dummy.host.com:8984_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41331/solr ready
   [junit4]   2> 1449522 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr
   [junit4]   2> 1449649 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1449654 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1450944 WARN  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[B501811370D20380]-SendThread(127.0.0.1:41331)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101eb6301ab0002 has expired
   [junit4]   2> 1450945 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1450945 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1450945 INFO  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.Overseer Overseer (id=72597879476912130-dummy.host.com:8984_solr-n_0000000000) closing
   [junit4]   2> 1450981 INFO  (OverseerStateUpdate-72597879476912130-dummy.host.com:8984_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr
   [junit4]   2> 1450997 ERROR (OverseerExitThread) [    ] o.a.s.c.Overseer could not read the data
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:306) ~[java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> 1451163 WARN  (zkCallback-1187-thread-1) [    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1451163 ERROR (OverseerAutoScalingTriggerThread-72597879476912130-dummy.host.com:8984_solr-n_0000000000) [    ] o.a.s.c.a.ScheduledTriggers Failed to add trigger .scheduled_maintenance
   [junit4]   2> org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.scheduled_maintenance
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:124) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:114) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:110) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueueFactory.makeQueue(ZkDistributedQueueFactory.java:36) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:49) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.<init>(ScheduledTriggers.java:528) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:207) [java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:234) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.scheduled_maintenance
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:122) ~[java/:?]
   [junit4]   2> ... 8 more
   [junit4]   2> 1451164 ERROR (OverseerAutoScalingTriggerThread-72597879476912130-dummy.host.com:8984_solr-n_0000000000) [    ] o.a.s.c.a.ScheduledTriggers Failed to add trigger .auto_add_replicas
   [junit4]   2> org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:124) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:114) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:110) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueueFactory.makeQueue(ZkDistributedQueueFactory.java:36) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:49) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.<init>(ScheduledTriggers.java:528) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:207) [java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:234) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:122) ~[java/:?]
   [junit4]   2> ... 8 more
   [junit4]   2> 1451165 WARN  (OverseerAutoScalingTriggerThread-72597879476912130-dummy.host.com:8984_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Error removing old nodeAdded markers
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.lang.Object.wait(Native Method) ~[?:1.8.0_172]
   [junit4]   2> at java.lang.Object.wait(Object.java:502) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1533) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:328) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328) ~[java/:?]
   [junit4]   2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.listData(ZkDistribStateManager.java:67) ~[java/:?]
   [junit4]   2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.listData(ZkDistribStateManager.java:77) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:264) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> 1451165 WARN  (OverseerAutoScalingTriggerThread-72597879476912130-dummy.host.com:8984_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed, exiting.
   [junit4]   2> 1451165 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1451269 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1451269 INFO  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1451269 INFO  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 1451381 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1451381 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1453081 WARN  (zkConnectionManagerCallback-1188-thread-1-SendThread(127.0.0.1:41331)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101eb6301ab0004 has expired
   [junit4]   2> 1453081 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1453081 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1453081 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1453110 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1453110 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1453110 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 1453182 ERROR (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.refreshAliases(ZkStateReader.java:1660)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:451)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 1453182 ERROR (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:417)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.refreshAliases(ZkStateReader.java:1660)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:451)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> ... 10 more
   [junit4]   2>
   [junit4]   2> 1453182 WARN  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 1453183 INFO  (zkConnectionManagerCallback-1188-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient Connected:true
   [junit4]   2> 1453218 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1453218 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1454323 WARN  (zkConnectionManagerCallback-1188-thread-1-SendThread(127.0.0.1:41331)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101eb6301ab0005 has expired
   [junit4]   2> 1454324 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1454324 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1454324 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1454383 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1454384 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1454384 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 1454509 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1454509 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1455619 WARN  (zkConnectionManagerCallback-1188-thread-1-SendThread(127.0.0.1:41331)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101eb6301ab0006 has expired
   [junit4]   2> 1455622 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1455623 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1455623 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1455679 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1455679 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1455679 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 1455729 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2>
   [junit4]   2> 1455729 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:417)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> ... 5 more
   [junit4]   2>
   [junit4]   2> 1455729 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 1455729 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient Connected:true
   [junit4]   2> 1455806 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1455806 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1457284 WARN  (zkConnectionManagerCallback-1188-thread-1-SendThread(127.0.0.1:41331)) [    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101eb6301ab0007 has expired
   [junit4]   2> 1457284 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 1457284 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 1457284 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 1457385 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2>
   [junit4]   2> 1457385 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:417)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> ... 5 more
   [junit4]   2>
   [junit4]   2> 1457385 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 1457385 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient Connected:false
   [junit4]   2> 1457386 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2>
   [junit4]   2> 1457386 ERROR (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:417)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
   [junit4]   2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
   [junit4]   2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:440)
   [junit4]   2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:332)
   [junit4]   2> ... 5 more
   [junit4]   2>
   [junit4]   2> 1457386 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 1457397 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1457401 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 1457402 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient Connected:true
   [junit4]   2> 1457402 INFO  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 1457519 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c78a71 name: ZooKeeperConnection Watcher:127.0.0.1:41331/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1457519 WARN  (zkConnectionManagerCallback-1188-thread-1-EventThread) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnec

[...truncated too long message...]

e] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1301708384
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1301708384
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1301708384

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null772850386
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null772850386
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null772850386

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 119ms :: artifacts dl 28ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 292 minutes 53 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 57 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/57/

1 tests failed.
FAILED:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings

Error Message:
end() called in wrong state=END!

Stack Trace:
java.lang.IllegalStateException: end() called in wrong state=END!
        at __randomizedtesting.SeedInfo.seed([8AC98B28DE6C27D3:E092343987220720]:0)
        at org.apache.lucene.analysis.MockTokenizer.fail(MockTokenizer.java:126)
        at org.apache.lucene.analysis.MockTokenizer.end(MockTokenizer.java:320)
        at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
        at org.apache.lucene.analysis.ValidatingTokenFilter.end(ValidatingTokenFilter.java:132)
        at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
        at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter.end(ConditionalTokenFilter.java:103)
        at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter$OneTimeWrapper.end(ConditionalTokenFilter.java:68)
        at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter.end(ConditionalTokenFilter.java:105)
        at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
        at org.apache.lucene.analysis.ValidatingTokenFilter.end(ValidatingTokenFilter.java:132)
        at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:442)
        at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:546)
        at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:872)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 2596 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> Exception from random analyzer:
   [junit4]   2> charfilters=
   [junit4]   2>   org.apache.lucene.analysis.charfilter.HTMLStripCharFilter(java.io.StringReader@3d36f105, [<EMOJI>, <NUM>, <HANGUL>])
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.MockTokenizer(initial state: 0
   [junit4]   2> state 0 [reject]:
   [junit4]   2>  a -> 1
   [junit4]   2>  b -> 2
   [junit4]   2>  f -> 3
   [junit4]   2>  i -> 4
   [junit4]   2>  n -> 5
   [junit4]   2>  o -> 6
   [junit4]   2>  s -> 7
   [junit4]   2>  t -> 8
   [junit4]   2>  w -> 9
   [junit4]   2> state 1 [accept]:
   [junit4]   2>  n -> 10
   [junit4]   2>  r -> 11
   [junit4]   2>  s -> 12
   [junit4]   2>  t -> 13
   [junit4]   2> state 2 [reject]:
   [junit4]   2>  e -> 14
   [junit4]   2>  u -> 15
   [junit4]   2>  y -> 16
   [junit4]   2> state 3 [reject]:
   [junit4]   2>  o -> 17
   [junit4]   2> state 4 [reject]:
   [junit4]   2>  f -> 18
   [junit4]   2>  n -> 19
   [junit4]   2>  s -> 20
   [junit4]   2>  t -> 21
   [junit4]   2> state 5 [reject]:
   [junit4]   2>  o -> 22
   [junit4]   2> state 6 [reject]:
   [junit4]   2>  f -> 23
   [junit4]   2>  n -> 24
   [junit4]   2>  r -> 25
   [junit4]   2> state 7 [reject]:
   [junit4]   2>  u -> 26
   [junit4]   2> state 8 [reject]:
   [junit4]   2>  h -> 27
   [junit4]   2>  o -> 28
   [junit4]   2> state 9 [reject]:
   [junit4]   2>  a -> 29
   [junit4]   2>  i -> 30
   [junit4]   2> state 10 [accept]:
   [junit4]   2>  d -> 31
   [junit4]   2> state 11 [reject]:
   [junit4]   2>  e -> 32
   [junit4]   2> state 12 [accept]:
   [junit4]   2> state 13 [accept]:
   [junit4]   2> state 14 [accept]:
   [junit4]   2> state 15 [reject]:
   [junit4]   2>  t -> 33
   [junit4]   2> state 16 [accept]:
   [junit4]   2> state 17 [reject]:
   [junit4]   2>  r -> 34
   [junit4]   2> state 18 [accept]:
   [junit4]   2> state 19 [accept]:
   [junit4]   2>  t -> 35
   [junit4]   2> state 20 [accept]:
   [junit4]   2> state 21 [accept]:
   [junit4]   2> state 22 [accept]:
   [junit4]   2>  t -> 36
   [junit4]   2> state 23 [accept]:
   [junit4]   2> state 24 [accept]:
   [junit4]   2> state 25 [accept]:
   [junit4]   2> state 26 [reject]:
   [junit4]   2>  c -> 37
   [junit4]   2> state 27 [reject]:
   [junit4]   2>  a -> 38
   [junit4]   2>  e -> 39
   [junit4]   2>  i -> 40
   [junit4]   2> state 28 [accept]:
   [junit4]   2> state 29 [reject]:
   [junit4]   2>  s -> 41
   [junit4]   2> state 30 [reject]:
   [junit4]   2>  l -> 42
   [junit4]   2>  t -> 43
   [junit4]   2> state 31 [accept]:
   [junit4]   2> state 32 [accept]:
   [junit4]   2> state 33 [accept]:
   [junit4]   2> state 34 [accept]:
   [junit4]   2> state 35 [reject]:
   [junit4]   2>  o -> 44
   [junit4]   2> state 36 [accept]:
   [junit4]   2> state 37 [reject]:
   [junit4]   2>  h -> 45
   [junit4]   2> state 38 [reject]:
   [junit4]   2>  t -> 46
   [junit4]   2> state 39 [accept]:
   [junit4]   2>  i -> 47
   [junit4]   2>  n -> 48
   [junit4]   2>  r -> 49
   [junit4]   2>  s -> 50
   [junit4]   2>  y -> 51
   [junit4]   2> state 40 [reject]:
   [junit4]   2>  s -> 52
   [junit4]   2> state 41 [accept]:
   [junit4]   2> state 42 [reject]:
   [junit4]   2>  l -> 53
   [junit4]   2> state 43 [reject]:
   [junit4]   2>  h -> 54
   [junit4]   2> state 44 [accept]:
   [junit4]   2> state 45 [accept]:
   [junit4]   2> state 46 [accept]:
   [junit4]   2> state 47 [reject]:
   [junit4]   2>  r -> 55
   [junit4]   2> state 48 [accept]:
   [junit4]   2> state 49 [reject]:
   [junit4]   2>  e -> 56
   [junit4]   2> state 50 [reject]:
   [junit4]   2>  e -> 57
   [junit4]   2> state 51 [accept]:
   [junit4]   2> state 52 [accept]:
   [junit4]   2> state 53 [accept]:
   [junit4]   2> state 54 [accept]:
   [junit4]   2> state 55 [accept]:
   [junit4]   2> state 56 [accept]:
   [junit4]   2> state 57 [accept]:
   [junit4]   2> , true, 0)
   [junit4]   2> filters=ConditionalTokenFilter:
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChainsWithLargeStrings -Dtests.seed=8AC98B28DE6C27D3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=fr-CH -Dtests.timezone=Pacific/Niue -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.77s J2 | TestRandomChains.testRandomChainsWithLargeStrings <<<
   [junit4]    > Throwable #1: java.lang.IllegalStateException: end() called in wrong state=END!
   [junit4]    > at __randomizedtesting.SeedInfo.seed([8AC98B28DE6C27D3:E092343987220720]:0)
   [junit4]    > at org.apache.lucene.analysis.MockTokenizer.fail(MockTokenizer.java:126)
   [junit4]    > at org.apache.lucene.analysis.MockTokenizer.end(MockTokenizer.java:320)
   [junit4]    > at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
   [junit4]    > at org.apache.lucene.analysis.ValidatingTokenFilter.end(ValidatingTokenFilter.java:132)
   [junit4]    > at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
   [junit4]    > at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter.end(ConditionalTokenFilter.java:103)
   [junit4]    > at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter$OneTimeWrapper.end(ConditionalTokenFilter.java:68)
   [junit4]    > at org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter.end(ConditionalTokenFilter.java:105)
   [junit4]    > at org.apache.lucene.analysis.TokenFilter.end(TokenFilter.java:46)
   [junit4]    > at org.apache.lucene.analysis.ValidatingTokenFilter.end(ValidatingTokenFilter.java:132)
   [junit4]    > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:442)
   [junit4]    > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:546)
   [junit4]    > at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:872)
   [junit4]    > at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {dummy=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=1911, maxMBSortInHeap=6.371349932607919, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=Pacific/Niue
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=239384648,total=316145664
   [junit4]   2> NOTE: All tests run in this JVM: [TestSoraniStemFilter, TestNorwegianMinimalStemFilterFactory, TestTwoFold, TestPortugueseStemFilter, TestGalicianAnalyzer, HTMLStripCharFilterTest, TestPatternReplaceFilter, TestFrenchLightStemFilter, TestCharTokenizers, TestScandinavianFoldingFilter, TestComplexPrefix, TestMorph, TestRussianAnalyzer, TestArabicFilters, TestThaiTokenizerFactory, TestScandinavianNormalizationFilterFactory, TestLengthFilterFactory, TestDelimitedPayloadTokenFilterFactory, TestDutchAnalyzer, TestUAX29URLEmailTokenizer, TestConv, TestSnowballPorterFilterFactory, TestGermanStemFilterFactory, TestHunspellStemFilter, TestCJKBigramFilterFactory, TestRussianLightStemFilterFactory, TestGreekLowerCaseFilterFactory, TestRandomChains]
   [junit4] Completed [92/288 (1!)] on J2 in 6.37s, 2 tests, 1 error <<< FAILURES!

[...truncated 62712 lines...]
[asciidoctor:convert] asciidoctor: ERROR: about-this-guide.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
[asciidoctor:convert] asciidoctor: ERROR: solr-glossary.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
     [java] Processed 2194 links (1751 relative) to 2949 anchors in 228 files
     [echo] Validated Links & Anchors via: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-ref-guide/bare-bones-html/

-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec]
     [exec] Crawl/parse...
     [exec]
     [exec] Verify...
     [echo] Checking for malformed docs...

jar-checksums:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

common.resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null203876255
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null203876255
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null203876255

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null143367115
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null143367115
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null143367115

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 80ms :: artifacts dl 6ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 161 minutes 47 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 58 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/58/

2 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testDeleteNode

Error Message:
unexpected DELETENODE status: {responseHeader={status=0,QTime=5},status={state=notfound,msg=Did not find [search_rate_trigger3/5e1193b96c99cfT20t96hcplzrc0kzt4bkja07l1/0] in any tasks queue}}

Stack Trace:
java.lang.AssertionError: unexpected DELETENODE status: {responseHeader={status=0,QTime=5},status={state=notfound,msg=Did not find [search_rate_trigger3/5e1193b96c99cfT20t96hcplzrc0kzt4bkja07l1/0] in any tasks queue}}
        at __randomizedtesting.SeedInfo.seed([5E14A061AE531331:7C866EE399999C4C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.lambda$testDeleteNode$6(SearchRateTriggerIntegrationTest.java:668)
        at java.util.ArrayList.forEach(ArrayList.java:1257)
        at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testDeleteNode(SearchRateTriggerIntegrationTest.java:660)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<8> but was:<10>

Stack Trace:
java.lang.AssertionError: expected:<8> but was:<10>
        at __randomizedtesting.SeedInfo.seed([5E14A061AE531331:33E8049C141BEC36]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:354)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13584 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/init-core-data-001
   [junit4]   2> 1605452 WARN  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1605452 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1605454 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1605455 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1605455 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 5 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001
   [junit4]   2> 1605455 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1605456 INFO  (Thread-2134) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1605456 INFO  (Thread-2134) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1605457 ERROR (Thread-2134) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1605556 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:59637
   [junit4]   2> 1605562 INFO  (zkConnectionManagerCallback-2851-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605569 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.Server jetty-9.4.10.v20180503; built: 2018-05-03T15:56:21.710Z; git: daa59876e6f384329b122929e70a80934569428c; jvm 1.8.0_172-b11
   [junit4]   2> 1605570 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.Server jetty-9.4.10.v20180503; built: 2018-05-03T15:56:21.710Z; git: daa59876e6f384329b122929e70a80934569428c; jvm 1.8.0_172-b11
   [junit4]   2> 1605577 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1605577 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1605577 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1605577 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24b35e12{/solr,null,AVAILABLE}
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7053552b{SSL,[ssl, http/1.1]}{127.0.0.1:54181}
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.e.j.s.Server Started @1605690ms
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54181}
   [junit4]   2> 1605579 ERROR (jetty-launcher-2848-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1605579 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-13T15:41:00.226Z
   [junit4]   2> 1605583 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.Server jetty-9.4.10.v20180503; built: 2018-05-03T15:56:21.710Z; git: daa59876e6f384329b122929e70a80934569428c; jvm 1.8.0_172-b11
   [junit4]   2> 1605583 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.Server jetty-9.4.10.v20180503; built: 2018-05-03T15:56:21.710Z; git: daa59876e6f384329b122929e70a80934569428c; jvm 1.8.0_172-b11
   [junit4]   2> 1605584 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.Server jetty-9.4.10.v20180503; built: 2018-05-03T15:56:21.710Z; git: daa59876e6f384329b122929e70a80934569428c; jvm 1.8.0_172-b11
   [junit4]   2> 1605584 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1605584 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1605584 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1605585 INFO  (zkConnectionManagerCallback-2853-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605585 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1605588 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1605588 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1605588 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1605589 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e3696e3{/solr,null,AVAILABLE}
   [junit4]   2> 1605589 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2092b777{SSL,[ssl, http/1.1]}{127.0.0.1:47805}
   [junit4]   2> 1605589 INFO  (jetty-launcher-2848-thread-2) [    ] o.e.j.s.Server Started @1605701ms
   [junit4]   2> 1605589 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47805}
   [junit4]   2> 1605590 ERROR (jetty-launcher-2848-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1605590 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1605590 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1605590 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1605590 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1605590 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-13T15:41:00.237Z
   [junit4]   2> 1605593 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a90cfb8{/solr,null,AVAILABLE}
   [junit4]   2> 1605593 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@85021e7{SSL,[ssl, http/1.1]}{127.0.0.1:57779}
   [junit4]   2> 1605593 INFO  (jetty-launcher-2848-thread-3) [    ] o.e.j.s.Server Started @1605704ms
   [junit4]   2> 1605593 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57779}
   [junit4]   2> 1605594 ERROR (jetty-launcher-2848-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1605594 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1605594 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1605594 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1605594 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1605594 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-13T15:41:00.241Z
   [junit4]   2> 1605596 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1605596 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1605596 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1605596 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a8c9b92{/solr,null,AVAILABLE}
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@38760d0e{SSL,[ssl, http/1.1]}{127.0.0.1:39954}
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.e.j.s.Server Started @1605708ms
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39954}
   [junit4]   2> 1605597 ERROR (jetty-launcher-2848-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1605597 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-13T15:41:00.244Z
   [junit4]   2> 1605603 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1605603 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1605603 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1605603 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@47bd7c1f{/solr,null,AVAILABLE}
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7f255eb2{SSL,[ssl, http/1.1]}{127.0.0.1:33014}
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.e.j.s.Server Started @1605715ms
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33014}
   [junit4]   2> 1605604 ERROR (jetty-launcher-2848-thread-5) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1605604 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-05-13T15:41:00.251Z
   [junit4]   2> 1605607 INFO  (zkConnectionManagerCallback-2857-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605607 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1605607 INFO  (zkConnectionManagerCallback-2855-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605608 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1605628 INFO  (zkConnectionManagerCallback-2859-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605628 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1605638 INFO  (jetty-launcher-2848-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59637/solr
   [junit4]   2> 1605639 INFO  (zkConnectionManagerCallback-2861-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605639 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1605640 INFO  (jetty-launcher-2848-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59637/solr
   [junit4]   2> 1605650 INFO  (jetty-launcher-2848-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59637/solr
   [junit4]   2> 1605651 INFO  (zkConnectionManagerCallback-2869-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605655 INFO  (jetty-launcher-2848-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59637/solr
   [junit4]   2> 1605657 INFO  (jetty-launcher-2848-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59637/solr
   [junit4]   2> 1605658 INFO  (zkConnectionManagerCallback-2866-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605659 INFO  (zkConnectionManagerCallback-2874-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605661 INFO  (zkConnectionManagerCallback-2881-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605664 INFO  (zkConnectionManagerCallback-2878-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605664 INFO  (zkConnectionManagerCallback-2885-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605664 INFO  (zkConnectionManagerCallback-2883-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605707 INFO  (zkConnectionManagerCallback-2887-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605708 INFO  (zkConnectionManagerCallback-2889-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605711 INFO  (zkConnectionManagerCallback-2895-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605739 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1605739 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1605739 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1605745 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57779_solr
   [junit4]   2> 1605747 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47805_solr
   [junit4]   2> 1605747 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33014_solr
   [junit4]   2> 1605748 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.Overseer Overseer (id=73792853437382669-127.0.0.1:57779_solr-n_0000000000) starting
   [junit4]   2> 1605752 INFO  (zkCallback-2894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1605761 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1605774 INFO  (zkCallback-2888-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1605795 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1605797 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1605799 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1605799 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54181_solr
   [junit4]   2> 1605824 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1605826 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1605826 INFO  (zkCallback-2894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1605826 INFO  (zkCallback-2888-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1605827 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1605827 INFO  (zkCallback-2882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1605828 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1605828 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39954_solr
   [junit4]   2> 1605829 INFO  (zkCallback-2888-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1605829 INFO  (zkCallback-2894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1605829 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1605844 INFO  (zkConnectionManagerCallback-2906-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1605844 INFO  (zkCallback-2882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1605852 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1605853 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1605872 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1605887 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1605887 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57779_solr
   [junit4]   2> 1605888 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605888 INFO  (zkCallback-2882-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605888 INFO  (zkCallback-2894-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605894 INFO  (zkCallback-2888-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605894 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605894 INFO  (zkCallback-2905-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1605908 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 1605916 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1605917 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 1605917 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1605924 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:47805_solr, 127.0.0.1:39954_solr, 127.0.0.1:54181_solr, 127.0.0.1:33014_solr, 127.0.0.1:57779_solr]
   [junit4]   2> 1605936 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1605936 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1605937 DEBUG (ScheduledTrigger-3793-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5
   [junit4]   2> 1606100 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33014.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606146 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33014.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606146 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33014.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606147 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node5/.
   [junit4]   2> 1606204 INFO  (zkConnectionManagerCallback-2911-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606220 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606221 INFO  (jetty-launcher-2848-thread-5) [n:127.0.0.1:33014_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1606283 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54181.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606297 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54181.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606297 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54181.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606317 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39954.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606324 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node1/.
   [junit4]   2> 1606350 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39954.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606350 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39954.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606352 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node3/.
   [junit4]   2> 1606370 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57779.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606383 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57779.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606383 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57779.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606384 INFO  (zkConnectionManagerCallback-2916-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606385 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606385 INFO  (jetty-launcher-2848-thread-3) [n:127.0.0.1:57779_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node4/.
   [junit4]   2> 1606386 INFO  (jetty-launcher-2848-thread-1) [n:127.0.0.1:54181_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1606389 INFO  (zkConnectionManagerCallback-2921-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606390 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606391 INFO  (jetty-launcher-2848-thread-4) [n:127.0.0.1:39954_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1606683 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47805.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606696 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47805.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606696 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47805.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1606697 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node2/.
   [junit4]   2> 1606701 INFO  (zkConnectionManagerCallback-2926-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606702 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606703 INFO  (jetty-launcher-2848-thread-2) [n:127.0.0.1:47805_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1606768 INFO  (zkConnectionManagerCallback-2928-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606771 INFO  (zkConnectionManagerCallback-2933-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1606772 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606773 INFO  (SUITE-SearchRateTriggerIntegrationTest-seed#[5E14A061AE531331]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59637/solr ready
   [junit4]   2> 1606839 INFO  (qtp1758755827-10146) [n:127.0.0.1:47805_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1606875 INFO  (qtp2002743592-10170) [n:127.0.0.1:33014_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1606883 INFO  (qtp1832386451-10161) [n:127.0.0.1:39954_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1606902 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1606933 INFO  (qtp571615543-10138) [n:127.0.0.1:57779_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
   [junit4]   2> 1606933 DEBUG (qtp1832386451-10160) [n:127.0.0.1:39954_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1606934 DEBUG (zkCallback-2905-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1606935 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1606935 INFO  (qtp1832386451-10160) [n:127.0.0.1:39954_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=126
   [junit4]   2> 1606939 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1606939 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1606940 DEBUG (ScheduledTrigger-3793-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 5
   [junit4]   2> 1606991 INFO  (TEST-SearchRateTriggerIntegrationTest.testDeleteNode-seed#[5E14A061AE531331]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteNode
   [junit4]   2> 1606992 INFO  (TEST-SearchRateTriggerIntegrationTest.testDeleteNode-seed#[5E14A061AE531331]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1606993 INFO  (TEST-SearchRateTriggerIntegrationTest.testDeleteNode-seed#[5E14A061AE531331]) [    ] o.a.s.c.a.SearchRateTriggerIntegrationTest /autoscaling.json reset, new znode version 3
   [junit4]   2> 1606993 DEBUG (zkCallback-2905-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1606993 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1606995 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1606995 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1606995 DEBUG (OverseerAutoScalingTriggerThread-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1612004 INFO  (qtp1758755827-10121) [n:127.0.0.1:47805_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=deleteNode_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1612016 INFO  (OverseerThreadFactory-3795-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection deleteNode_collection
   [junit4]   2> 1612127 INFO  (OverseerStateUpdate-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteNode_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteNode_collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:57779/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1612130 INFO  (OverseerStateUpdate-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteNode_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteNode_collection_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39954/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"}
   [junit4]   2> 1612356 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr    x:deleteNode_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteNode_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1612369 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr    x:deleteNode_collection_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteNode_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1613422 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1613423 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1613431 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.s.IndexSchema [deleteNode_collection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1613431 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.s.IndexSchema [deleteNode_collection_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 1613435 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1613435 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1613435 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'deleteNode_collection_shard1_replica_n2' using configuration from collection deleteNode_collection, trusted=true
   [junit4]   2> 1613435 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'deleteNode_collection_shard1_replica_n1' using configuration from collection deleteNode_collection, trusted=true
   [junit4]   2> 1613435 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39954.solr.core.deleteNode_collection.shard1.replica_n2' (registry 'solr.core.deleteNode_collection.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1613435 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57779.solr.core.deleteNode_collection.shard1.replica_n1' (registry 'solr.core.deleteNode_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1613435 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1613435 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1613436 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.SolrCore [[deleteNode_collection_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node3/deleteNode_collection_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node3/./deleteNode_collection_shard1_replica_n2/data/]
   [junit4]   2> 1613436 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SolrCore [[deleteNode_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node4/deleteNode_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node4/./deleteNode_collection_shard1_replica_n1/data/]
   [junit4]   2> 1613516 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1613516 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1613516 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1613516 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1613518 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1613518 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1613518 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1613518 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1613520 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1895a82f[deleteNode_collection_shard1_replica_n2] main]
   [junit4]   2> 1613520 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@311ae89a[deleteNode_collection_shard1_replica_n1] main]
   [junit4]   2> 1613521 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1613522 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1613522 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1613522 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1613523 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1613523 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1613523 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1600364025657425920
   [junit4]   2> 1613524 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1600364025658474496
   [junit4]   2> 1613526 INFO  (searcherExecutor-3806-thread-1-processing-n:127.0.0.1:39954_solr x:deleteNode_collection_shard1_replica_n2 c:deleteNode_collection s:shard1 r:core_node4) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.SolrCore [deleteNode_collection_shard1_replica_n2] Registered new searcher Searcher@1895a82f[deleteNode_collection_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1613527 INFO  (searcherExecutor-3807-thread-1-processing-n:127.0.0.1:57779_solr x:deleteNode_collection_shard1_replica_n1 c:deleteNode_collection s:shard1 r:core_node3) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SolrCore [deleteNode_collection_shard1_replica_n1] Registered new searcher Searcher@311ae89a[deleteNode_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1613535 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteNode_collection/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 1613538 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteNode_collection/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
   [junit4]   2> 1613542 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1613542 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1613543 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:57779/solr/deleteNode_collection_shard1_replica_n1/
   [junit4]   2> 1613543 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=deleteNode_collection_shard1_replica_n1 url=https://127.0.0.1:57779/solr START replicas=[https://127.0.0.1:39954/solr/deleteNode_collection_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 1613547 INFO  (qtp1832386451-10159) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.c.S.Request [deleteNode_collection_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1613844 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=deleteNode_collection_shard1_replica_n1 url=https://127.0.0.1:57779/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 1613844 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1613844 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1613844 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1613852 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:57779/solr/deleteNode_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1613969 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1613974 INFO  (qtp571615543-10132) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deleteNode_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1617
   [junit4]   2> 1614072 INFO  (zkCallback-2888-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1614545 INFO  (qtp1832386451-10158) [n:127.0.0.1:39954_solr c:deleteNode_collection s:shard1 r:core_node4 x:deleteNode_collection_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteNode_collection_shard1_replica_n2&action=CREATE&numShards=1&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2176
   [junit4]   2> 1614549 INFO  (qtp1758755827-10121) [n:127.0.0.1:47805_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1614644 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1614644 INFO  (zkCallback-2888-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1615551 INFO  (qtp1758755827-10121) [n:127.0.0.1:47805_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=deleteNode_collection&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3546
   [junit4]   2> 1615552 INFO  (qtp1758755827-10146) [n:127.0.0.1:47805_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params action=ADDREPLICA&collection=deleteNode_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1615555 INFO  (OverseerCollectionConfigSetProcessor-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1615558 INFO  (OverseerThreadFactory-3795-thread-2) [ c:deleteNode_collection s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:47805_solr for creating new replica
   [junit4]   2> 1615563 INFO  (OverseerStateUpdate-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"deleteNode_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteNode_collection_shard1_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47805/solr",
   [junit4]   2>   "node_name":"127.0.0.1:47805_solr",
   [junit4]   2>   "type":"NRT"}
   [junit4]   2> 1615665 INFO  (zkCallback-2888-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1615665 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1615768 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr    x:deleteNode_collection_shard1_replica_n5] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=deleteNode_collection_shard1_replica_n5&action=CREATE&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1615769 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr    x:deleteNode_collection_shard1_replica_n5] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1615782 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1615790 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.s.IndexSchema [deleteNode_collection_shard1_replica_n5] Schema name=minimal
   [junit4]   2> 1615793 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1615793 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'deleteNode_collection_shard1_replica_n5' using configuration from collection deleteNode_collection, trusted=true
   [junit4]   2> 1615794 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47805.solr.core.deleteNode_collection.shard1.replica_n5' (registry 'solr.core.deleteNode_collection.shard1.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1615794 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1615794 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.SolrCore [[deleteNode_collection_shard1_replica_n5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node2/deleteNode_collection_shard1_replica_n5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node2/./deleteNode_collection_shard1_replica_n5/data/]
   [junit4]   2> 1615879 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1615879 INFO  (zkCallback-2888-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1615889 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1615889 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1615891 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1615891 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1615893 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@38bcf3ed[deleteNode_collection_shard1_replica_n5] main]
   [junit4]   2> 1615895 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1615895 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1615896 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1615897 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1600364028146745344
   [junit4]   2> 1615901 INFO  (searcherExecutor-3816-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.SolrCore [deleteNode_collection_shard1_replica_n5] Registered new searcher Searcher@38bcf3ed[deleteNode_collection_shard1_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1615904 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteNode_collection/terms/shard1 to Terms{values={core_node6=0, core_node3=0, core_node4=0}, version=2}
   [junit4]   2> 1615906 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.ZkController Core needs to recover:deleteNode_collection_shard1_replica_n5
   [junit4]   2> 1615907 INFO  (updateExecutor-2870-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1615913 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 1615914 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 1615914 INFO  (qtp1758755827-10145) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=deleteNode_collection_shard1_replica_n5&action=CREATE&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=145
   [junit4]   2> 1615917 INFO  (qtp1758755827-10146) [n:127.0.0.1:47805_solr c:deleteNode_collection   ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=ADDREPLICA&collection=deleteNode_collection&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=364
   [junit4]   2> 1615918 INFO  (qtp1758755827-10147) [n:127.0.0.1:47805_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params action=ADDREPLICA&collection=deleteNode_collection&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1615921 INFO  (qtp571615543-10136) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.S.Request [deleteNode_collection_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1615921 INFO  (qtp571615543-10136) [n:127.0.0.1:57779_solr c:deleteNode_collection s:shard1 r:core_node3 x:deleteNode_collection_shard1_replica_n1] o.a.s.c.S.Request [deleteNode_collection_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1615921 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[deleteNode_collection_shard1_replica_n5]
   [junit4]   2> 1615921 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 1615921 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.RecoveryStrategy Publishing state of core [deleteNode_collection_shard1_replica_n5] as recovering, leader is [https://127.0.0.1:57779/solr/deleteNode_collection_shard1_replica_n1/] and I am [https://127.0.0.1:47805/solr/deleteNode_collection_shard1_replica_n5/]
   [junit4]   2> 1615925 INFO  (OverseerCollectionConfigSetProcessor-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1615933 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteNode_collection/terms/shard1 to Terms{values={core_node6=0, core_node6_recovering=0, core_node3=0, core_node4=0}, version=3}
   [junit4]   2> 1615936 INFO  (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:47805_solr x:deleteNode_collection_shard1_replica_n5 c:deleteNode_collection s:shard1 r:core_node6) [n:127.0.0.1:47805_solr c:deleteNode_collection s:shard1 r:core_node6 x:deleteNode_collection_shard1_replica_n5] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:57779/solr]; [WaitForState: action=PREPRECOVERY&core=deleteNode_collection_shard1_replica_n1&nodeName=127.0.0.1:47805_solr&coreNodeName=core_node6&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 1615936 INFO  (OverseerThreadFactory-3795-thread-3) [ c:deleteNode_collection s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:54181_solr for creating new replica
   [junit4]   2> 1615939 INFO  (OverseerStateUpdate-73792853437382669-127.0.0.1:57779_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"deleteNode_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteNode_collection_shard1_replica_t7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:54181/solr",
   [junit4]   2>   "node_name":"127.0.0.1:54181_solr",
   [junit4]   2>   "type":"TLOG"}
   [junit4]   2> 1615948 INFO  (qtp571615543-10138) [n:127.0.0.1:57779_solr    x:deleteNode_collection_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node6, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 1615950 INFO  (qtp571615543-10138) [n:127.0.0.1:57779_solr    x:deleteNode_collection_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=deleteNode_collection, shard=shard1, thisCore=deleteNode_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:47805_solr, coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: core_node6:{"core":"deleteNode_collection_shard1_replica_n5","base_url":"https://127.0.0.1:47805/solr","node_name":"127.0.0.1:47805_solr","state":"down","type":"NRT","force_set_state":"false"}
   [junit4]   2> 1616044 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616044 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616045 INFO  (zkCallback-2888-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616048 INFO  (zkCallback-2886-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616149 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr    x:deleteNode_collection_shard1_replica_t7] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&name=deleteNode_collection_shard1_replica_t7&action=CREATE&collection=deleteNode_collection&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 1616162 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1616170 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.s.IndexSchema [deleteNode_collection_shard1_replica_t7] Schema name=minimal
   [junit4]   2> 1616174 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1616174 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.CoreContainer Creating SolrCore 'deleteNode_collection_shard1_replica_t7' using configuration from collection deleteNode_collection, trusted=true
   [junit4]   2> 1616177 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54181.solr.core.deleteNode_collection.shard1.replica_t7' (registry 'solr.core.deleteNode_collection.shard1.replica_t7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@428d74d2
   [junit4]   2> 1616178 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1616178 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.SolrCore [[deleteNode_collection_shard1_replica_t7] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node1/deleteNode_collection_shard1_replica_t7], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerIntegrationTest_5E14A061AE531331-001/tempDir-001/node1/./deleteNode_collection_shard1_replica_t7/data/]
   [junit4]   2> 1616250 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1616250 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1616252 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1616252 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1616254 INFO  (zkCallback-2888-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616254 INFO  (zkCallback-2886-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616254 INFO  (zkCallback-2886-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616254 INFO  (zkCallback-2884-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deleteNode_collection/state.json] for collection [deleteNode_collection] has occurred - updating... (live nodes size: [5])
   [junit4]   2> 1616256 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a975371[deleteNode_collection_shard1_replica_t7] main]
   [junit4]   2> 1616258 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1616258 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1616259 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1616260 INFO  (searcherExecutor-3821-thread-1-processing-n:127.0.0.1:54181_solr x:deleteNode_collection_shard1_replica_t7 c:deleteNode_collection s:shard1 r:core_node8) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.SolrCore [deleteNode_collection_shard1_replica_t7] Registered new searcher Searcher@7a975371[deleteNode_collection_shard1_replica_t7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1616260 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1600364028527378432
   [junit4]   2> 1616267 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteNode_collection/terms/shard1 to Terms{values={core_node6=0, core_node8=0, core_node6_recovering=0, core_node3=0, core_node4=0}, version=4}
   [junit4]   2> 1616269 INFO  (qtp277687652-10127) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.ZkController Core needs to recover:deleteNode_collection_shard1_replica_t7
   [junit4]   2> 1616270 INFO  (updateExecutor-2862-thread-1-processing-n:127.0.0.1:54181_solr x:deleteNode_collection_shard1_replica_t7 c:deleteNode_collection s:shard1 r:core_node8) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1616270 INFO  (recoveryExecutor-2867-thread-1-processing-n:127.0.0.1:54181_solr x:deleteNode_collection_shard1_replica_t7 c:deleteNode_collection s:shard1 r:core_node8) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 1616273 INFO  (recoveryExecutor-2867-thread-1-processing-n:127.0.0.1:54181_solr x:deleteNode_collection_shard1_replica_t7 c:deleteNode_collection s:shard1 r:core_node8) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 1616273 INFO  (recoveryExecutor-2867-thread-1-processing-n:127.0.0.1:54181_solr x:deleteNode_collection_shard1_replica_t7 c:deleteNode_collection s:shard1 r:core_node8) [n:127.0.0.1:54181_solr c:deleteNode_collection s:shard1 r:core_node8 x:deleteNode_collection_shard1_replica_t7] o.a.s.c.ZkController deleteNode_collection_shard1_replica_t7 stopping background replication from leader
   [junit4]   2> 1616275 INFO  (qtp277687652-10

[...truncated too long message...]

lowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null370765764
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null370765764
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null370765764

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1999070075
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1999070075
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1999070075

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 76ms :: artifacts dl 6ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 164 minutes 8 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 59 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/59/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
        at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
        at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
        at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([C328FE39FAA33E5B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
        at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest.testSimple

Error Message:
Waiting for collection testSimple2 null Live Nodes: [127.0.0.1:44173_solr, 127.0.0.1:58743_solr] Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/25)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node2":{           "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node2/data/",           "base_url":"http://127.0.0.1:44173/solr",           "node_name":"127.0.0.1:44173_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node2/data/tlog",           "core":"testSimple2_shard1_replica_n1",           "shared_storage":"true",           "state":"active",           "leader":"true"},         "core_node5":{           "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node5/data/",           "base_url":"http://127.0.0.1:36725/solr",           "node_name":"127.0.0.1:36725_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node5/data/tlog",           "core":"testSimple2_shard1_replica_n3",           "shared_storage":"true",           "state":"down"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node7":{           "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node7/data/",           "base_url":"http://127.0.0.1:44173/solr",           "node_name":"127.0.0.1:44173_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node7/data/tlog",           "core":"testSimple2_shard2_replica_n4",           "shared_storage":"true",           "state":"active",           "leader":"true"},         "core_node8":{           "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node8/data/",           "base_url":"http://127.0.0.1:36725/solr",           "node_name":"127.0.0.1:36725_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node8/data/tlog",           "core":"testSimple2_shard2_replica_n6",           "shared_storage":"true",           "state":"down"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Waiting for collection testSimple2
null
Live Nodes: [127.0.0.1:44173_solr, 127.0.0.1:58743_solr]
Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/25)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node2/data/",
          "base_url":"http://127.0.0.1:44173/solr",
          "node_name":"127.0.0.1:44173_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node2/data/tlog",
          "core":"testSimple2_shard1_replica_n1",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"},
        "core_node5":{
          "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node5/data/",
          "base_url":"http://127.0.0.1:36725/solr",
          "node_name":"127.0.0.1:36725_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node5/data/tlog",
          "core":"testSimple2_shard1_replica_n3",
          "shared_storage":"true",
          "state":"down"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node7/data/",
          "base_url":"http://127.0.0.1:44173/solr",
          "node_name":"127.0.0.1:44173_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node7/data/tlog",
          "core":"testSimple2_shard2_replica_n4",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"},
        "core_node8":{
          "dataDir":"hdfs://localhost:35226/data/testSimple2/core_node8/data/",
          "base_url":"http://127.0.0.1:36725/solr",
          "node_name":"127.0.0.1:36725_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35226/data/testSimple2/core_node8/data/tlog",
          "core":"testSimple2_shard2_replica_n6",
          "shared_storage":"true",
          "state":"down"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
        at __randomizedtesting.SeedInfo.seed([C328FE39FAA33E5B:FB9BDAC7DD50EA8A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:278)
        at org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple(AutoAddReplicasIntegrationTest.java:129)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testMergeIntegration

Error Message:
did not finish processing in time

Stack Trace:
java.lang.AssertionError: did not finish processing in time
        at __randomizedtesting.SeedInfo.seed([C328FE39FAA33E5B:9091BC8918B2ABA1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest.testMergeIntegration(IndexSizeTriggerTest.java:404)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([C328FE39FAA33E5B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([C328FE39FAA33E5B]:0)




Build Log:
[...truncated 12851 lines...]
   [junit4] Suite: org.apache.solr.cloud.ZkControllerTest
   [junit4]   2> 888857 INFO  (SUITE-ZkControllerTest-seed#[C328FE39FAA33E5B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ZkControllerTest_C328FE39FAA33E5B-001/init-core-data-001
   [junit4]   2> 888857 WARN  (SUITE-ZkControllerTest-seed#[C328FE39FAA33E5B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=214 numCloses=214
   [junit4]   2> 888868 INFO  (SUITE-ZkControllerTest-seed#[C328FE39FAA33E5B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 888870 INFO  (SUITE-ZkControllerTest-seed#[C328FE39FAA33E5B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 888873 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeNameUrlConversion
   [junit4]   2> 888874 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 888885 INFO  (Thread-882) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 888885 INFO  (Thread-882) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 888912 ERROR (Thread-882) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 888985 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.ZkTestServer start zk server on port:43690
   [junit4]   2> 889216 INFO  (zkConnectionManagerCallback-1370-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889244 INFO  (zkConnectionManagerCallback-1372-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889268 INFO  (zkConnectionManagerCallback-1374-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889408 ERROR (zkCallback-1373-thread-1) [    ] o.a.s.c.c.ZkStateReader Error reading cluster properties from zookeeper
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterprops.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.loadClusterProperties(ZkStateReader.java:989) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkStateReader.lambda$new$5(ZkStateReader.java:981) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:269) ~[java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> 889416 ERROR (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 889416 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43690 43690
   [junit4]   2> 889518 INFO  (Thread-882) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43690 43690
   [junit4]   2> 889519 WARN  (Thread-882) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/aliases.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/clusterstate.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/live_nodes
   [junit4]   2> 2 /solr/collections
   [junit4]   2>
   [junit4]   2> 889520 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[C328FE39FAA33E5B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeNameUrlConversion
   [junit4]   2> 889522 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReadConfigName
   [junit4]   2> 889522 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 889522 INFO  (Thread-883) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 889522 INFO  (Thread-883) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 889530 ERROR (Thread-883) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 889676 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.ZkTestServer start zk server on port:38491
   [junit4]   2> 889736 INFO  (zkConnectionManagerCallback-1380-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889756 INFO  (zkConnectionManagerCallback-1382-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889791 INFO  (zkConnectionManagerCallback-1384-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889801 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@537b6faa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 889845 INFO  (zkConnectionManagerCallback-1388-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 889995 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 889995 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 889996 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73802051630596099-127.0.0.1:8983_solr-n_0000000000) starting
   [junit4]   2> 890096 INFO  (zkConnectionManagerCallback-1395-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 890098 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38491/solr ready
   [junit4]   2> 890123 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
   [junit4]   2> 890132 INFO  (OverseerStateUpdate-73802051630596099-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 890134 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73802051630596099-127.0.0.1:8983_solr-n_0000000000) closing
   [junit4]   2> 890156 INFO  (zkCallback-1394-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 890156 INFO  (OverseerStateUpdate-73802051630596099-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:8983_solr
   [junit4]   2> 890185 WARN  (OverseerExitThread) [    ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
   [junit4]   2> 890233 WARN  (OverseerAutoScalingTriggerThread-73802051630596099-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed, exiting.
   [junit4]   2> 890234 WARN  (ScheduledTrigger-1787-thread-1) [    ] o.a.s.c.a.TriggerBase Exception updating trigger state '/autoscaling/triggerState/.auto_add_replicas'
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.lang.Object.wait(Native Method) ~[?:1.8.0_172]
   [junit4]   2> at java.lang.Object.wait(Object.java:502) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1100) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316) ~[java/:?]
   [junit4]   2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:58) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:234) [java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:596) [java/:?]
   [junit4]   2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> 890237 INFO  (OverseerExitThread) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 890238 INFO  (OverseerExitThread) [    ] o.a.s.c.Overseer Overseer (id=73802051630596099-127.0.0.1:8983_solr-n_0000000001) starting
   [junit4]   2> 890277 WARN  (OverseerAutoScalingTriggerThread-73802051630596099-127.0.0.1:8983_solr-n_0000000001) [    ] o.a.s.c.a.OverseerTriggerThread Solr cannot talk to ZK, exiting OverseerTriggerThread main queue loop
   [junit4]   2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
   [junit4]   2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183) ~[java/:?]
   [junit4]   2> at org.apache.solr.client.solrj.cloud.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:83) ~[java/:?]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:131) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176) ~[java/:?]
   [junit4]   2> ... 3 more
   [junit4]   2> 890280 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=605559004
   [junit4]   2> 890280 ERROR (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrCores No transient handler has been defined. Check solr.xml to see if an attempt to provide a custom TransientSolrCoreCacheFactory was done incorrectly since the default should have been used otherwise.
   [junit4]   2> 890280 ERROR (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 890281 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38491 38491
   [junit4]   2> 890281 ERROR (OverseerCollectionConfigSetProcessor-73802051630596099-127.0.0.1:8983_solr-n_0000000001) [    ] o.a.s.c.OverseerTaskProcessor :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/collection-queue-work
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
   [junit4]   2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
   [junit4]   2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1535)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:328)
   [junit4]   2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
   [junit4]   2> at org.apache.solr.cloud.ZkDistributedQueue.fetchZkChildren(ZkDistributedQueue.java:410)
   [junit4]   2> at org.apache.solr.cloud.OverseerTaskQueue.getTailId(OverseerTaskQueue.java:272)
   [junit4]   2> at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:163)
   [junit4]   2> at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>
   [junit4]   2> 890282 INFO  (OverseerCollectionConfigSetProcessor-73802051630596099-127.0.0.1:8983_solr-n_0000000001) [    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=73802051630596099-127.0.0.1:8983_solr-n_0000000001) am no longer a leader.
   [junit4]   2> 890377 INFO  (Thread-883) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38491 38491
   [junit4]   2> 890377 WARN  (Thread-883) [    ] o.a.s.c.ZkTestServer Watch limit violations:
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/aliases.json
   [junit4]   2> 2 /solr/clusterprops.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/clusterstate.json
   [junit4]   2>
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2>
   [junit4]   2> 2 /solr/overseer/queue
   [junit4]   2> 2 /solr/live_nodes
   [junit4]   2> 2 /solr/collections
   [junit4]   2> 2 /solr/overseer/queue-work
   [junit4]   2>
   [junit4]   2> 890378 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testReadConfigName
   [junit4]   2> 890380 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Starting testPublishAndWaitForDownStates
   [junit4]   2> 890381 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 890388 INFO  (Thread-884) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 890388 INFO  (Thread-884) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 890412 ERROR (Thread-884) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 890488 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer start zk server on port:44338
   [junit4]   2> 890497 INFO  (zkConnectionManagerCallback-1397-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 890499 INFO  (zkConnectionManagerCallback-1399-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 890505 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@537b6faa, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 890507 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$318/1301159414@dd7eccf
   [junit4]   2> 890509 INFO  (zkConnectionManagerCallback-1403-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 890757 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
   [junit4]   2> 890758 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 890758 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 890758 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 890758 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 890759 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 890762 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 890762 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73802051687809026-127.0.0.1:8983_solr-n_0000000000
   [junit4]   2> 890762 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 890763 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73802051687809026-127.0.0.1:8983_solr-n_0000000000) starting
   [junit4]   2> 890786 INFO  (zkConnectionManagerCallback-1410-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 890787 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
   [junit4]   2> 890787 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 890788 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 890788 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 890788 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 890789 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44338/solr ready
   [junit4]   2> 890789 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 890792 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 890793 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 890793 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 890803 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
   [junit4]   2> 890803 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 890803 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 890805 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
   [junit4]   2> 890808 INFO  (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 890809 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
   [junit4]   2> 890809 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 890812 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 890814 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 890814 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 890816 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 890816 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 890816 INFO  (zkCallback-1409-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 890817 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[C328FE39FAA33E5B]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:8983_solr as DOWN
   [junit4]   2> 890829 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 890831 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890832 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.scheduled_maintenance
   [junit4]   2> 890834 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 890835 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 890840 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 890841 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
   [junit4]   2>   "operation":"create",
   [junit4]   2>   "node_name":"127.0.0.1:8983_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "name":"testPublishAndWaitForDownStates",
   [junit4]   2>   "stateFormat":"2"} current state version: 0
   [junit4]   2> 890841 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: testPublishAndWaitForDownStates
   [junit4]   2> 890841 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890842 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: create/delete: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890842 INFO  (zkCallback-1402-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/testPublishAndWaitForDownStates/state.json] for collection [testPublishAndWaitForDownStates] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 890842 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 890842 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890843 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host1",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "operation":"addreplica"} current state version: 0
   [junit4]   2> 890843 INFO  (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host1",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "operation":"addreplica"}
   [junit4]   2> 890844 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{}}
   [junit4]   2> 890844 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator New Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{"core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 890844 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host2",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "operation":"addreplica"} current state version: 0
   [junit4]   2> 890844 INFO  (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host2",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "operation":"addreplica"}
   [junit4]   2> 890845 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{"core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 890845 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator New Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{
   [junit4]   2>     "core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null},
   [junit4]   2>     "core_node22":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"down",
   [junit4]   2>       "node_name":"non_existent_host2",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 890845 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:8983_solr"} current state version: 0
   [junit4]   2> 890846 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:8983_solr
   [junit4]   2> 890846 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 890946 DEBUG (OverseerStateUpdate-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testPublishAndWaitForDownStates/state.json version: 0
   [junit4]   2> 890947 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890947 INFO  (zkCallback-1402-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testPublishAndWaitForDownStates/state.json] for collection [testPublishAndWaitForDownStates] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 890947 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 890947 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 890949 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 892794 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 893267 INFO  (OverseerStateUpdate-73802051630596099-127.0.0.1:8983_solr-n_0000000001) [    ] o.a.s.c.Overseer According to ZK I (id=73802051630596099-127.0.0.1:8983_solr-n_0000000001) am no longer a leader.
   [junit4]   2> 893267 INFO  (OverseerStateUpdate-73802051630596099-127.0.0.1:8983_solr-n_0000000001) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:8983_solr
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 894795 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 896796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 898796 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 898800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 900800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 900800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 900800 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 900801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 900801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 900801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 900801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 902801 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 904802 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 906803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 908803 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 908804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 910804 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 910805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 912805 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 912806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 914806 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 914807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 916807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 916807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 916807 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 916808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 916808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 916808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 916808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 918808 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 920810 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 920811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 922811 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 922812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 924812 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 924813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 926813 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 926814 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 928815 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 928816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 930816 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 930817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 932817 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 932818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 934818 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 936819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 938819 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 938820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 940820 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 940821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 942821 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 942822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 944822 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 944823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 946823 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 946824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 948824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 948824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 948824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 948824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 948824 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 948825 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 948825 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 950874 DEBUG (OverseerCollectionConfigSetProcessor-73802051687809026-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 950874 WARN  (TEST-Zk

[...truncated too long message...]

owed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null674525468
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null674525468
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null674525468

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1470687114
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1470687114
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null1470687114

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 74ms :: artifacts dl 9ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 238 minutes 45 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 60 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/60/

2 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at __randomizedtesting.SeedInfo.seed([AAB60BDC2D23D53B:F7FE1555E2E57374]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate(TestLargeCluster.java:579)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic

Error Message:
there should be new MOVERPLICA ops

Stack Trace:
java.lang.AssertionError: there should be new MOVERPLICA ops
        at __randomizedtesting.SeedInfo.seed([AAB60BDC2D23D53B:14C16C9F2FF5315]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic(TestLargeCluster.java:225)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13424 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
   [junit4]   2> 3208706 INFO  (SUITE-TestLargeCluster-seed#[AAB60BDC2D23D53B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_AAB60BDC2D23D53B-001/init-core-data-001
   [junit4]   2> 3208719 WARN  (SUITE-TestLargeCluster-seed#[AAB60BDC2D23D53B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 3208719 INFO  (SUITE-TestLargeCluster-seed#[AAB60BDC2D23D53B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3208721 INFO  (SUITE-TestLargeCluster-seed#[AAB60BDC2D23D53B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3208722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3208722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 3208723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 3208723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 3208729 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10043_solr, 127.0.0.1:10018_solr, 127.0.0.1:10057_solr, 127.0.0.1:10049_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10005_solr, 127.0.0.1:10013_solr, 127.0.0.1:10021_solr, 127.0.0.1:10058_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10022_solr, 127.0.0.1:10039_solr, 127.0.0.1:10033_solr, 127.0.0.1:10025_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10044_solr, 127.0.0.1:10081_solr, 127.0.0.1:10028_solr, 127.0.0.1:10036_solr, 127.0.0.1:10084_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10041_solr, 127.0.0.1:10079_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10054_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10015_solr, 127.0.0.1:10055_solr, 127.0.0.1:10069_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10062_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10045_solr, 127.0.0.1:10037_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10053_solr, 127.0.0.1:10083_solr, 127.0.0.1:10059_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10056_solr, 127.0.0.1:10064_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3208730 DEBUG (ScheduledTrigger-5640-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 85
   [junit4]   2> 3208730 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3208730 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 3208751 DEBUG (ScheduledTrigger-5640-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208761 INFO  (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
   [junit4]   2> 3208761 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 3208761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 3208761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3208761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3208761 INFO  (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 3208761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 3208761 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2034) ~[?:1.8.0_172]
   [junit4]   2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) [java/:?]
   [junit4]   2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 3208766 DEBUG (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3208766 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 3208767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 3208767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 3208767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 3208767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3208767 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3208768 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 3208797 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208817 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208837 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208857 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208869 DEBUG (simCloudManagerPool-5642-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3208869 DEBUG (simCloudManagerPool-5642-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 3208869 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3208870 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3208870 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3208873 DEBUG (simCloudManagerPool-5642-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3208873 DEBUG (simCloudManagerPool-5642-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 3208873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 3208873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10104_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 3208873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3208873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 3208885 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208885 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3208906 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208906 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3208986 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3208986 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209016 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209016 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209052 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209052 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209089 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209089 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209109 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209109 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209129 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209129 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209149 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209149 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209170 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209170 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209190 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209190 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209210 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209210 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209230 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209231 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209251 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209251 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209271 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209271 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209291 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209291 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209312 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209312 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209332 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209332 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209352 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209352 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209372 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209372 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209393 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209393 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209413 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209413 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209433 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209433 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209453 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209453 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209474 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209474 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209494 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209494 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209514 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209514 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209535 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209535 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209555 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209555 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209575 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209575 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209595 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209595 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209616 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209616 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209636 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209636 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209656 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209656 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209676 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209676 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209697 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209697 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209717 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209717 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209737 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209737 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209757 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209757 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209778 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209778 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209798 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209798 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209818 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209818 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209839 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209839 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209859 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209859 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209879 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209879 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209899 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209900 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209920 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209920 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209940 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209940 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209960 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209960 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3209981 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3209981 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210001 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210001 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210021 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210021 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210041 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210042 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210062 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210062 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210092 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210092 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210112 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210112 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210133 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210133 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210153 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210153 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210173 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210173 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210193 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210193 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210214 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210214 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210234 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210234 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210254 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210254 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210274 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210274 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210295 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210295 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210315 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210315 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210335 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210335 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210356 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210356 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210381 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210381 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210401 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210401 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210421 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210421 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210441 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210441 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210462 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210462 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210482 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210482 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210502 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210502 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210522 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210523 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210543 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210543 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210567 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210567 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210588 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210588 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210608 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210608 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210641 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210641 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210661 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210661 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210697 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210697 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210718 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210718 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210742 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210742 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210762 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210762 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210785 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210785 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210805 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210805 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210825 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210825 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210846 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210846 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210866 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210866 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210886 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210886 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210917 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210917 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210941 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210941 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210961 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210961 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3210982 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3210982 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211006 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211006 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211043 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211043 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211063 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211063 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211095 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211096 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211116 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211116 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211152 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211153 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211189 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211189 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211225 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211226 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211246 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211246 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211266 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211266 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211286 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211286 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211307 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211307 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211327 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211327 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211342 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard20: {"core_node191":{
   [junit4]   2>     "core":"testNodeLost_shard20_replica_n191",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10025_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard20",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211342 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard10: {"core_node92":{
   [junit4]   2>     "core":"testNodeLost_shard10_replica_n92",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10102_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard10",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211342 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard11: {"core_node108":{
   [junit4]   2>     "core":"testNodeLost_shard11_replica_n108",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10058_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard11",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard12: {"core_node119":{
   [junit4]   2>     "core":"testNodeLost_shard12_replica_n119",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10099_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard12",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard2: {"core_node18":{
   [junit4]   2>     "core":"testNodeLost_shard2_replica_n18",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10092_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard13: {"core_node129":{
   [junit4]   2>     "core":"testNodeLost_shard13_replica_n129",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10060_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard13",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard3: {"core_node29":{
   [junit4]   2>     "core":"testNodeLost_shard3_replica_n29",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10060_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard3",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard14: {"core_node139":{
   [junit4]   2>     "core":"testNodeLost_shard14_replica_n139",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10096_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard14",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211343 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard15: {"core_node149":{
   [junit4]   2>     "core":"testNodeLost_shard15_replica_n149",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10009_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard15",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211364 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard4: {"core_node39":{
   [junit4]   2>     "core":"testNodeLost_shard4_replica_n39",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10096_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard4",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard17: {"core_node169":{
   [junit4]   2>     "core":"testNodeLost_shard17_replica_n169",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10008_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard17",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard18: {"core_node180":{
   [junit4]   2>     "core":"testNodeLost_shard18_replica_n180",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10018_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard18",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard19: {"core_node190":{
   [junit4]   2>     "core":"testNodeLost_shard19_replica_n190",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10090_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard19",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: {"core_node1":{
   [junit4]   2>     "core":"testNodeLost_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10069_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard6: {"core_node60":{
   [junit4]   2>     "core":"testNodeLost_shard6_replica_n60",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10064_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard6",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard7: {"core_node70":{
   [junit4]   2>     "core":"testNodeLost_shard7_replica_n70",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10067_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard7",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard8: {"core_node80":{
   [junit4]   2>     "core":"testNodeLost_shard8_replica_n80",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10018_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard8",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211426 DEBUG (simCloudManagerPool-5642-thread-3) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard9: {"core_node90":{
   [junit4]   2>     "core":"testNodeLost_shard9_replica_n90",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10090_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard9",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211425 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211425 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211444 DEBUG (simCloudManagerPool-5642-thread-200) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard5: {"core_node50":{
   [junit4]   2>     "core":"testNodeLost_shard5_replica_n50",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10012_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard5",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211445 DEBUG (simCloudManagerPool-5642-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard16: {"core_node159":{
   [junit4]   2>     "core":"testNodeLost_shard16_replica_n159",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10022_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard16",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211461 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211462 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211486 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
   [junit4]   2> 3211486 DEBUG (ScheduledTrigger-5643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
   [junit4]   2> 3211519 INFO  (TEST-TestLargeCluster.testNodeLost-seed#[AAB60BDC2D23D53B]) [    ] o.a.s.c.a.s.TestLargeCluster Ready after 4922 ms
   [junit4]   2> 3211526 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 99
   [junit4]   2> 3211526 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10082_solr
   [junit4]   2> 3211526 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 99
   [junit4]   2> 3211526 DEBUG (ScheduledTrigger-5643-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10082_solr
   [junit4]   2> 3211555 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 99
   [junit4]   2> 3211571 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 99
   [junit4]   2> 3211579 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 99
   [junit4]   2> 3211591 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 99
   [junit4]   2> 3211599 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 99
   [junit4]   2> 3211611 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 99
   [junit4]   2> 3211619 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 99
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 98
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 98
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10018_solr
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10082_solr]
   [junit4]   2> 3211640 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1fd70f9e0196a3Tcbxuq1csavgppe1ywi2v9qqat",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":8962186353415843,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[8962186353415843],
   [junit4]   2>     "nodeNames":["127.0.0.1:10082_solr"]}}
   [junit4]   2> 3211661 DEBUG (ScheduledTrigger-5643-thread-3) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
   [junit4]   2> 3211662 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1fd70f9e0196a3Tcbxuq1csavgppe1ywi2v9qqat",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":8962186353415843,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[8962186353415843],
   [junit4]   2>     "_enqueue_time_":8962193099886693,
   [junit4]   2>     "nodeNames":["127.0.0.1:10082_solr"]}}
   [junit4]   2> 3211662 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"1fd70f9e0196a3Tcbxuq1csavgppe1ywi2v9qqat",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":8962186353415843,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[8962186353415843],
   [junit4]   2>     "_enqueue_time_":8962193099886693,
   [junit4]   2>     "nodeNames":["127.0.0.1:10082_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
   [junit4]   2> 3211678 DEBUG (simCloudManagerPool-5642-thread-41) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard18: {"core_node179":{
   [junit4]   2>     "core":"testNodeLost_shard18_replica_n179",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10011_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard18",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3211678 DEBUG (simCloudManagerPool-5642-thread-43) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard8: {"core_node72":{
   [junit4]   2>     "core":"testNodeLost_shard8_replica_n72",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10029_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard8",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3212180 INFO  (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node189
   [junit4]   2> 3212310 INFO  (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node89
   [junit4]   2> 3212310 INFO  (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 8962225576 sessionWrapper.createTime 8962194962787293, this.sessionWrapper.createTime 8962194962787293
   [junit4]   2> 3212310 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"1fd70f9e0196a3Tcbxuq1csavgppe1ywi2v9qqat",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":8962186353415843,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[8962186353415843],
   [junit4]   2>     "_enqueue_time_":8962193099886693,
   [junit4]   2>     "nodeNames":["127.0.0.1:10082_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@d64494e, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@3ae769b1], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
   [junit4]   2> 3212311 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node189
   [junit4]   2> 3212413 DEBUG (simCloudManagerPool-5642-thread-185) [    ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node202":{
   [junit4]   2>     "core":"testNodeLost_shard19_replica_n201",
   [junit4]   2>     "shard":"shard19",
   [junit4]   2>     "collection":"testNodeLost",
   [junit4]   2>     "node_name":"127.0.0.1:10072_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 3212539 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node89
   [junit4]   2> 3212647 DEBUG (simCloudManagerPool-5642-thread-200) [    ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node204":{
   [junit4]   2>     "core":"testNodeLost_shard9_replica_n203",
   [junit4]   2>     "shard":"shard9",
   [junit4]   2>     "collection":"testNodeLost",
   [junit4]   2>     "node_name":"127.0.0.1:10056_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 3212863 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3212863 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
   [junit4]   2> 3212863 DEBUG (AutoscalingActionExecutor-5644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 60044 ms for event id=1fd70f9e0196a3Tcbxuq1csavgppe1ywi2v9qqat
   [junit4]   2> 3212945 DEBUG (simCloudManagerPool-5642-thread-111) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard14: {"core_node138":{
   [junit4]   2>     "core":"testNodeLost_shard14_replica_n138",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10055_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard14",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3212946 DEBUG (simCloudManagerPool-5642-thread-111) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard4: {"core_node40":{
   [junit4]   2>     "core":"testNodeLost_shard4_replica_n40",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "SEARCHER.searcher.maxDoc":0,
   [junit4]   2>     "SEARCHER.searcher.deletedDocs":0,
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "node_name":"127.0.0.1:10006_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "SEARCHER.searcher.numDocs":0,
   [junit4]   2>     "shard":"shard4",
   [junit4]   2>     "collection":"testNodeLost"}}
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 87
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10104_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10020_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10065_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10066_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10051_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr
   [junit4]   2> 3212972 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10096_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 87
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10104_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10020_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10065_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10066_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10035_solr
   [junit4]   2> 3212973 DEBUG (ScheduledTrigger-5643-thr

[...truncated too long message...]

figure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1605961234
     [copy] Copying 39 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1605961234
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/null1605961234

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null33474232
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null33474232
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null33474232

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 51ms :: artifacts dl 1ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 305 minutes 51 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)


---------------------------------------------------------------------
To unsubscribe, e-mail: [hidden email]
For additional commands, e-mail: [hidden email]
Reply | Threaded
Open this post in threaded view
|

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 61 - Still Unstable

Apache Jenkins Server-2
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/61/

11 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:46031/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:37179/solr/MoveReplicaHDFSTest_failed_coll_true]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:46031/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:37179/solr/MoveReplicaHDFSTest_failed_coll_true]
        at __randomizedtesting.SeedInfo.seed([8448C61E6428EE4D:2E8515ECD3FB3B9D]:0)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:993)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:288)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46031/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
        ... 46 more


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
        at __randomizedtesting.SeedInfo.seed([8448C61E6428EE4D:C1CF9C4CAD483B5]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertTrue(Assert.java:54)
        at org.apache.solr.cloud.MultiThreadedOCPTest.testFillWorkQueue(MultiThreadedOCPTest.java:113)
        at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:68)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup

Error Message:
should be at least one inactive event

Stack Trace:
java.lang.AssertionError: should be at least one inactive event
        at __randomizedtesting.SeedInfo.seed([8448C61E6428EE4D:9964066C056BC946]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup(ScheduledMaintenanceTriggerTest.java:224)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testBelowSearchRate

Error Message:
[Op{action=DELETEREPLICA, hints={COLL_SHARD=[{   "first":"belowRate_collection",   "second":"shard1"}], REPLICA=[core_node6]}}, Op{action=DELETEREPLICA, hints={COLL_SHARD=[{   "first":"belowRate_collection",   "second":"shard1"}], REPLICA=[core_node8]}}, Op{action=DELETEREPLICA, hints={COLL_SHARD=[{   "first":"belowRate_collection",   "second":"shard1"}], REPLICA=[core_node10]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:37749_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:35431_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:38327_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:45021_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:34118_solr]}}] expected:<7> but was:<8>

Stack Trace:
java.lang.AssertionError: [Op{action=DELETEREPLICA, hints={COLL_SHARD=[{
  "first":"belowRate_collection",
  "second":"shard1"}], REPLICA=[core_node6]}}, Op{action=DELETEREPLICA, hints={COLL_SHARD=[{
  "first":"belowRate_collection",
  "second":"shard1"}], REPLICA=[core_node8]}}, Op{action=DELETEREPLICA, hints={COLL_SHARD=[{
  "first":"belowRate_collection",
  "second":"shard1"}], REPLICA=[core_node10]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:37749_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:35431_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:38327_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:45021_solr]}}, Op{action=NONE, hints={SRC_NODE=[127.0.0.1:34118_solr]}}] expected:<7> but was:<8>
        at __randomizedtesting.SeedInfo.seed([8448C61E6428EE4D:EEE5607E3CC06B5E]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testBelowSearchRate(SearchRateTriggerIntegrationTest.java:394)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost

Error Message:
last state: DocCollection(testNodeLost//clusterstate.json/261)={   "replicationFactor":"10",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"5",   "autoAddReplicas":"false",   "nrtReplicas":"10",   "tlogReplicas":"0",   "autoCreated":"true",   "shards":{     "shard20":{       "replicas":{         "core_node191":{           "core":"testNodeLost_shard20_replica_n191",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10081_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node200":{           "core":"testNodeLost_shard20_replica_n200",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10015_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node199":{           "core":"testNodeLost_shard20_replica_n199",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10121_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node198":{           "core":"testNodeLost_shard20_replica_n198",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10117_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node197":{           "core":"testNodeLost_shard20_replica_n197",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10037_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node196":{           "core":"testNodeLost_shard20_replica_n196",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10101_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node195":{           "core":"testNodeLost_shard20_replica_n195",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10016_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node194":{           "core":"testNodeLost_shard20_replica_n194",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10026_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node193":{           "core":"testNodeLost_shard20_replica_n193",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10098_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node192":{           "core":"testNodeLost_shard20_replica_n192",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10126_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"73330000-7fffffff",       "state":"active"},     "shard10":{       "replicas":{         "core_node92":{           "core":"testNodeLost_shard10_replica_n92",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10126_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node91":{           "core":"testNodeLost_shard10_replica_n91",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10081_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node94":{           "core":"testNodeLost_shard10_replica_n94",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10026_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node93":{           "core":"testNodeLost_shard10_replica_n93",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10098_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node96":{           "core":"testNodeLost_shard10_replica_n96",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10101_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node95":{           "core":"testNodeLost_shard10_replica_n95",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10016_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node98":{           "core":"testNodeLost_shard10_replica_n98",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10117_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node100":{           "core":"testNodeLost_shard10_replica_n100",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10015_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node97":{           "core":"testNodeLost_shard10_replica_n97",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10037_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node99":{           "core":"testNodeLost_shard10_replica_n99",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10121_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"f3330000-ffffffff",       "state":"active"},     "shard11":{       "replicas":{         "core_node108":{           "core":"testNodeLost_shard11_replica_n108",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10084_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node107":{           "core":"testNodeLost_shard11_replica_n107",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10059_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node106":{           "core":"testNodeLost_shard11_replica_n106",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10050_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node105":{           "core":"testNodeLost_shard11_replica_n105",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10021_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node104":{           "core":"testNodeLost_shard11_replica_n104",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10034_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node103":{           "core":"testNodeLost_shard11_replica_n103",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10055_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node102":{           "core":"testNodeLost_shard11_replica_n102",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10027_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node101":{           "core":"testNodeLost_shard11_replica_n101",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10038_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node110":{           "core":"testNodeLost_shard11_replica_n110",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10103_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node109":{           "core":"testNodeLost_shard11_replica_n109",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10132_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"0-ccbffff",       "state":"active"},     "shard12":{       "replicas":{         "core_node119":{           "core":"testNodeLost_shard12_replica_n119",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10024_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node118":{           "core":"testNodeLost_shard12_replica_n118",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10106_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node117":{           "core":"testNodeLost_shard12_replica_n117",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10102_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node116":{           "core":"testNodeLost_shard12_replica_n116",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10096_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node115":{           "core":"testNodeLost_shard12_replica_n115",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10046_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node114":{           "core":"testNodeLost_shard12_replica_n114",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10020_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node113":{           "core":"testNodeLost_shard12_replica_n113",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10052_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node112":{           "core":"testNodeLost_shard12_replica_n112",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10072_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node111":{           "core":"testNodeLost_shard12_replica_n111",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10041_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node120":{           "core":"testNodeLost_shard12_replica_n120",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10012_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"ccc0000-1998ffff",       "state":"active"},     "shard13":{       "replicas":{         "core_node129":{           "core":"testNodeLost_shard13_replica_n129",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10053_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node206":{           "node_name":"127.0.0.1:10103_solr",           "core":"testNodeLost_shard13_replica_n205",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node128":{           "core":"testNodeLost_shard13_replica_n128",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10099_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node126":{           "core":"testNodeLost_shard13_replica_n126",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10062_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node125":{           "core":"testNodeLost_shard13_replica_n125",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10093_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node124":{           "core":"testNodeLost_shard13_replica_n124",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10087_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node123":{           "core":"testNodeLost_shard13_replica_n123",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10115_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node122":{           "core":"testNodeLost_shard13_replica_n122",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10073_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node121":{           "core":"testNodeLost_shard13_replica_n121",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10045_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node130":{           "core":"testNodeLost_shard13_replica_n130",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10029_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"19990000-2665ffff",       "state":"active"},     "shard2":{       "replicas":{         "core_node18":{           "core":"testNodeLost_shard2_replica_n18",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10106_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node17":{           "core":"testNodeLost_shard2_replica_n17",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10102_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node19":{           "core":"testNodeLost_shard2_replica_n19",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10024_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node20":{           "core":"testNodeLost_shard2_replica_n20",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10012_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node12":{           "core":"testNodeLost_shard2_replica_n12",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10072_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node11":{           "core":"testNodeLost_shard2_replica_n11",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10041_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node14":{           "core":"testNodeLost_shard2_replica_n14",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10020_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node13":{           "core":"testNodeLost_shard2_replica_n13",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10052_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node16":{           "core":"testNodeLost_shard2_replica_n16",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10096_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node15":{           "core":"testNodeLost_shard2_replica_n15",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10046_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"8ccc0000-9998ffff",       "state":"active"},     "shard3":{       "replicas":{         "core_node29":{           "core":"testNodeLost_shard3_replica_n29",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10053_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node28":{           "core":"testNodeLost_shard3_replica_n28",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10099_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node30":{           "core":"testNodeLost_shard3_replica_n30",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10029_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node21":{           "core":"testNodeLost_shard3_replica_n21",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10045_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node23":{           "core":"testNodeLost_shard3_replica_n23",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10115_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node22":{           "core":"testNodeLost_shard3_replica_n22",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10073_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node25":{           "core":"testNodeLost_shard3_replica_n25",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10093_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node24":{           "core":"testNodeLost_shard3_replica_n24",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10087_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node26":{           "core":"testNodeLost_shard3_replica_n26",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10062_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node208":{           "node_name":"127.0.0.1:10059_solr",           "core":"testNodeLost_shard3_replica_n207",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"}},       "range":"99990000-a665ffff",       "state":"active"},     "shard14":{       "replicas":{         "core_node139":{           "core":"testNodeLost_shard14_replica_n139",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10090_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node138":{           "core":"testNodeLost_shard14_replica_n138",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10127_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node137":{           "core":"testNodeLost_shard14_replica_n137",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10110_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node136":{           "core":"testNodeLost_shard14_replica_n136",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10067_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node135":{           "core":"testNodeLost_shard14_replica_n135",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10129_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node134":{           "core":"testNodeLost_shard14_replica_n134",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10100_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node133":{           "core":"testNodeLost_shard14_replica_n133",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10119_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node132":{           "core":"testNodeLost_shard14_replica_n132",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10056_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node131":{           "core":"testNodeLost_shard14_replica_n131",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10018_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node140":{           "core":"testNodeLost_shard14_replica_n140",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10085_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"26660000-3332ffff",       "state":"active"},     "shard15":{       "replicas":{         "core_node149":{           "core":"testNodeLost_shard15_replica_n149",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10086_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node148":{           "core":"testNodeLost_shard15_replica_n148",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10048_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node147":{           "core":"testNodeLost_shard15_replica_n147",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10133_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node146":{           "core":"testNodeLost_shard15_replica_n146",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10008_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node145":{           "core":"testNodeLost_shard15_replica_n145",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10064_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node144":{           "core":"testNodeLost_shard15_replica_n144",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10116_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node143":{           "core":"testNodeLost_shard15_replica_n143",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10094_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node142":{           "core":"testNodeLost_shard15_replica_n142",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10014_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node141":{           "core":"testNodeLost_shard15_replica_n141",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10006_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node150":{           "core":"testNodeLost_shard15_replica_n150",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10078_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"33330000-3fffffff",       "state":"active"},     "shard4":{       "replicas":{         "core_node39":{           "core":"testNodeLost_shard4_replica_n39",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10090_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node40":{           "core":"testNodeLost_shard4_replica_n40",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10085_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node32":{           "core":"testNodeLost_shard4_replica_n32",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10056_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node31":{           "core":"testNodeLost_shard4_replica_n31",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10018_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node34":{           "core":"testNodeLost_shard4_replica_n34",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10100_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node33":{           "core":"testNodeLost_shard4_replica_n33",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10119_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node36":{           "core":"testNodeLost_shard4_replica_n36",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10067_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node35":{           "core":"testNodeLost_shard4_replica_n35",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10129_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node38":{           "core":"testNodeLost_shard4_replica_n38",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10127_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node37":{           "core":"testNodeLost_shard4_replica_n37",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10110_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"a6660000-b332ffff",       "state":"active"},     "shard5":{       "replicas":{         "core_node50":{           "core":"testNodeLost_shard5_replica_n50",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10078_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node41":{           "core":"testNodeLost_shard5_replica_n41",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10006_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node43":{           "core":"testNodeLost_shard5_replica_n43",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10094_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node42":{           "core":"testNodeLost_shard5_replica_n42",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10014_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node45":{           "core":"testNodeLost_shard5_replica_n45",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10064_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node44":{           "core":"testNodeLost_shard5_replica_n44",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10116_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node47":{           "core":"testNodeLost_shard5_replica_n47",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10133_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node46":{           "core":"testNodeLost_shard5_replica_n46",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10008_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node49":{           "core":"testNodeLost_shard5_replica_n49",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10086_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node48":{           "core":"testNodeLost_shard5_replica_n48",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10048_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"b3330000-bfffffff",       "state":"active"},     "shard16":{       "replicas":{         "core_node159":{           "core":"testNodeLost_shard16_replica_n159",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10131_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node158":{           "core":"testNodeLost_shard16_replica_n158",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10031_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node157":{           "core":"testNodeLost_shard16_replica_n157",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10105_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node156":{           "core":"testNodeLost_shard16_replica_n156",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10079_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node155":{           "core":"testNodeLost_shard16_replica_n155",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10030_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node154":{           "core":"testNodeLost_shard16_replica_n154",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10123_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node153":{           "core":"testNodeLost_shard16_replica_n153",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10120_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node152":{           "core":"testNodeLost_shard16_replica_n152",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10066_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node151":{           "core":"testNodeLost_shard16_replica_n151",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10122_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node160":{           "core":"testNodeLost_shard16_replica_n160",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10023_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"40000000-4ccbffff",       "state":"active"},     "shard17":{       "replicas":{         "core_node169":{           "core":"testNodeLost_shard17_replica_n169",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10039_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node168":{           "core":"testNodeLost_shard17_replica_n168",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10019_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node167":{           "core":"testNodeLost_shard17_replica_n167",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10111_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node166":{           "core":"testNodeLost_shard17_replica_n166",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10080_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node165":{           "core":"testNodeLost_shard17_replica_n165",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10125_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node164":{           "core":"testNodeLost_shard17_replica_n164",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10047_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node163":{           "core":"testNodeLost_shard17_replica_n163",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10033_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node162":{           "core":"testNodeLost_shard17_replica_n162",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10044_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node161":{           "core":"testNodeLost_shard17_replica_n161",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10051_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node170":{           "core":"testNodeLost_shard17_replica_n170",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10091_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"4ccc0000-5998ffff",       "state":"active"},     "shard18":{       "replicas":{         "core_node180":{           "core":"testNodeLost_shard18_replica_n180",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10077_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node179":{           "core":"testNodeLost_shard18_replica_n179",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10040_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node178":{           "core":"testNodeLost_shard18_replica_n178",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10068_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node177":{           "core":"testNodeLost_shard18_replica_n177",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10097_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node176":{           "core":"testNodeLost_shard18_replica_n176",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10108_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node175":{           "core":"testNodeLost_shard18_replica_n175",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10010_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node174":{           "core":"testNodeLost_shard18_replica_n174",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10060_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node173":{           "core":"testNodeLost_shard18_replica_n173",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10036_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node172":{           "core":"testNodeLost_shard18_replica_n172",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10112_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node171":{           "core":"testNodeLost_shard18_replica_n171",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10092_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"59990000-6665ffff",       "state":"active"},     "shard19":{       "replicas":{         "core_node190":{           "core":"testNodeLost_shard19_replica_n190",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10042_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node202":{           "node_name":"127.0.0.1:10072_solr",           "core":"testNodeLost_shard19_replica_n201",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node189":{           "core":"testNodeLost_shard19_replica_n189",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10075_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node188":{           "core":"testNodeLost_shard19_replica_n188",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10058_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node187":{           "core":"testNodeLost_shard19_replica_n187",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10114_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node186":{           "core":"testNodeLost_shard19_replica_n186",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10011_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node184":{           "core":"testNodeLost_shard19_replica_n184",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10035_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node183":{           "core":"testNodeLost_shard19_replica_n183",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10007_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node182":{           "core":"testNodeLost_shard19_replica_n182",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10017_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node181":{           "core":"testNodeLost_shard19_replica_n181",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10061_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"66660000